{ "baseline": null, "config": { "excluded_operators": [ "QuantizeLinear", "DynamicQuantizeLinear" ], "metamorphic": true, "optimizations": [ "disabled", "all" ], "repeats": 7, "seed": 20260906, "warmup": 2 }, "created_at": "2026-09-06T11:09:56.426798+00:00", "duplicates_removed": 0, "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "evidence_scope": "Local synthetic-graph CPU benchmark. A divergence is an investigation candidate, not an upstream bug or novelty claim.", "finding_graphs": 7, "results": [ { "artifacts": { "bundle": "observations/1d124a0942871adc9fee3d9ac0e3a740092e6ede01f33a61ef001360f1a2c6ce/reproducer.zip", "bundle_sha256": "a199ba0a36ae5acdd313ffbd153140a91400a87bf5f5bc29018953904434b27f", "case": "artifacts/aad9f7f1019aa44bf33264ffb62b6eab381bde07d0a2958f66bf7b82de659979/case.json", "inputs": "artifacts/aad9f7f1019aa44bf33264ffb62b6eab381bde07d0a2958f66bf7b82de659979/inputs.npz", "manifest": "artifacts/aad9f7f1019aa44bf33264ffb62b6eab381bde07d0a2958f66bf7b82de659979/manifest.json", "model": "artifacts/aad9f7f1019aa44bf33264ffb62b6eab381bde07d0a2958f66bf7b82de659979/model.onnx", "outputs": "observations/1d124a0942871adc9fee3d9ac0e3a740092e6ede01f33a61ef001360f1a2c6ce/outputs.npz" }, "candidate_group": "889f7c80dbc7517f0111497671261aa681ceb276389b5c021abce992f2c21622", "case_id": "aad9f7f1019aa44bf33264ffb62b6eab381bde07d0a2958f66bf7b82de659979", "category": "metrics", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "cosine_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "backend": "reference", "name": "cosine_range", "passed": true }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 8, 1 ], "backend": "reference", "elements": 8, "expected_dtype": "float16", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "cosine_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "backend": "ort", "name": "cosine_range", "passed": true }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 8, 1 ], "backend": "ort", "elements": 8, "expected_dtype": "float16", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } } ], "comparison": { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, "comparison_key": "58a237d67c5736da2edfe41bd2f8b01e82f76fa31e3858d7e246577c4553d84d", "contract": "cosine", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "target": { "dtype": "float16", "shape": [ 4, 17 ] }, "x": { "dtype": "float16", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "CosineSimilarity/constant/float16/4x17/seed-20260906", "observation_id": "1d124a0942871adc9fee3d9ac0e3a740092e6ede01f33a61ef001360f1a2c6ce", "operation": "CosineSimilarity", "operators": [ "Div", "Max", "Mul", "ReduceSum", "Sqrt" ], "opset": 19, "optimization": "all", "output_hash": "7d7c8145bb2ff21a9e91bda1a1277f081e0dd2396b40a78d7729246e9965612a", "outputs": {}, "parameters": { "epsilon": 1e-06 }, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.007417, "p95_ms": 0.0079669, "repeats": 7, "samples_ms": [ 0.0075, 0.008167, 0.007417, 0.007458, 0.0065, 0.006542, 0.006292 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.062792, "p95_ms": 0.0639464, "repeats": 7, "samples_ms": [ 0.064334, 0.063042, 0.062834, 0.062458, 0.061916, 0.062792, 0.061791 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 0.0005, "rtol": 0.005 }, "variant": "constant", "warnings": [] }, { "artifacts": { "bundle": "observations/89f0536d1ad097f404809a854bda6dd1216c091018715bb8dfb55db893fc06dd/reproducer.zip", "bundle_sha256": "08720a6843a4121ef325560b34197706198e98ee877292eea37f2370195907d3", "case": "artifacts/aad9f7f1019aa44bf33264ffb62b6eab381bde07d0a2958f66bf7b82de659979/case.json", "inputs": "artifacts/aad9f7f1019aa44bf33264ffb62b6eab381bde07d0a2958f66bf7b82de659979/inputs.npz", "manifest": "artifacts/aad9f7f1019aa44bf33264ffb62b6eab381bde07d0a2958f66bf7b82de659979/manifest.json", "model": "artifacts/aad9f7f1019aa44bf33264ffb62b6eab381bde07d0a2958f66bf7b82de659979/model.onnx", "outputs": "observations/89f0536d1ad097f404809a854bda6dd1216c091018715bb8dfb55db893fc06dd/outputs.npz" }, "candidate_group": "889f7c80dbc7517f0111497671261aa681ceb276389b5c021abce992f2c21622", "case_id": "aad9f7f1019aa44bf33264ffb62b6eab381bde07d0a2958f66bf7b82de659979", "category": "metrics", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "cosine_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "backend": "reference", "name": "cosine_range", "passed": true }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 8, 1 ], "backend": "reference", "elements": 8, "expected_dtype": "float16", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "cosine_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "backend": "ort", "name": "cosine_range", "passed": true }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 8, 1 ], "backend": "ort", "elements": 8, "expected_dtype": "float16", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } } ], "comparison": { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, "comparison_key": "da6001459f81ae387800578111e0c3842df62c64525b63498c631f4305668fc2", "contract": "cosine", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "target": { "dtype": "float16", "shape": [ 4, 17 ] }, "x": { "dtype": "float16", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "CosineSimilarity/constant/float16/4x17/seed-20260906", "observation_id": "89f0536d1ad097f404809a854bda6dd1216c091018715bb8dfb55db893fc06dd", "operation": "CosineSimilarity", "operators": [ "Div", "Max", "Mul", "ReduceSum", "Sqrt" ], "opset": 19, "optimization": "disabled", "output_hash": "7d7c8145bb2ff21a9e91bda1a1277f081e0dd2396b40a78d7729246e9965612a", "outputs": {}, "parameters": { "epsilon": 1e-06 }, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.00725, "p95_ms": 0.009157899999999998, "repeats": 7, "samples_ms": [ 0.009583, 0.008166, 0.007333, 0.00725, 0.006792, 0.007125, 0.006334 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.063167, "p95_ms": 0.06370870000000001, "repeats": 7, "samples_ms": [ 0.063708, 0.063709, 0.063167, 0.062625, 0.062583, 0.063209, 0.062833 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 0.0005, "rtol": 0.005 }, "variant": "constant", "warnings": [] }, { "artifacts": { "bundle": "observations/83e1a7450a8ffe78c5cdd930d7938f1819395c18b18a801545de36e181298358/reproducer.zip", "bundle_sha256": "ea32ce6503cace9fe912e46720890f464da853542170e981ae9e90a10141e408", "case": "artifacts/45f5b7ef4839b16f9f06cac2d744e34b5b09c1a0d325150d401b87dad108644a/case.json", "inputs": "artifacts/45f5b7ef4839b16f9f06cac2d744e34b5b09c1a0d325150d401b87dad108644a/inputs.npz", "manifest": "artifacts/45f5b7ef4839b16f9f06cac2d744e34b5b09c1a0d325150d401b87dad108644a/manifest.json", "model": "artifacts/45f5b7ef4839b16f9f06cac2d744e34b5b09c1a0d325150d401b87dad108644a/model.onnx", "outputs": "observations/83e1a7450a8ffe78c5cdd930d7938f1819395c18b18a801545de36e181298358/outputs.npz" }, "candidate_group": "336a655e64e5fec07886628d7ae79875e9491e382d785ad205a401aaba305c5a", "case_id": "45f5b7ef4839b16f9f06cac2d744e34b5b09c1a0d325150d401b87dad108644a", "category": "metrics", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 5.960464477539063e-08, "max_budget_ratio": 0.005418631376608828, "max_rel": 5.960497517959426e-08, "max_ulp": 1, "mismatched": 0, "name": "cosine_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "reference", "name": "cosine_range", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 1 ], "backend": "reference", "elements": 8, "expected_dtype": "float32", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 1.7881393432617188e-07, "max_budget_ratio": 0.016255894129826483, "max_rel": 1.7881492553878278e-07, "max_ulp": 3, "mismatched": 0, "name": "cosine_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "cosine_range", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 1.1920928955078125e-07, "max_budget_ratio": 0.0108372645149119, "max_rel": 1.1920997167571073e-07, "max_ulp": 2, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 1.1920928955078125e-07, "max_budget_ratio": 0.0108372645149119, "max_rel": 1.1920997167571073e-07, "max_ulp": 2, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 1 ], "backend": "ort", "elements": 8, "expected_dtype": "float32", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } } ], "comparison": { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 1.1920928955078125e-07, "max_budget_ratio": 0.010837263340449007, "max_rel": 1.1920995746469508e-07, "max_ulp": 2, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, "comparison_key": "71c082bc232c53bfebcebf437a71b33c2536a4a6d0eabce6dbb250fc17a37100", "contract": "cosine", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "target": { "dtype": "float32", "shape": [ 4, 17 ] }, "x": { "dtype": "float32", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "CosineSimilarity/constant/float32/4x17/seed-20260906", "observation_id": "83e1a7450a8ffe78c5cdd930d7938f1819395c18b18a801545de36e181298358", "operation": "CosineSimilarity", "operators": [ "Div", "Max", "Mul", "ReduceSum", "Sqrt" ], "opset": 19, "optimization": "all", "output_hash": "cfa02269f64315d4449acc84754c860abd238861d379a59498fc09dfb4e58411", "outputs": {}, "parameters": { "epsilon": 1e-06 }, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.006167, "p95_ms": 0.00675, "repeats": 7, "samples_ms": [ 0.00675, 0.00675, 0.0065, 0.005833, 0.005833, 0.005459, 0.006167 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.06125, "p95_ms": 0.0619751, "repeats": 7, "samples_ms": [ 0.062, 0.0615, 0.061125, 0.06125, 0.061917, 0.060833, 0.059875 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-06, "rtol": 1e-05 }, "variant": "constant", "warnings": [] }, { "artifacts": { "bundle": "observations/65558486cf0b696980f0cd378b843f3a399f7a048dc28f38eb45156bd962b8cd/reproducer.zip", "bundle_sha256": "8a32f527ac46d7d15df4d10454cd17461463059f7660a29d4b53a92161b4b506", "case": "artifacts/45f5b7ef4839b16f9f06cac2d744e34b5b09c1a0d325150d401b87dad108644a/case.json", "inputs": "artifacts/45f5b7ef4839b16f9f06cac2d744e34b5b09c1a0d325150d401b87dad108644a/inputs.npz", "manifest": "artifacts/45f5b7ef4839b16f9f06cac2d744e34b5b09c1a0d325150d401b87dad108644a/manifest.json", "model": "artifacts/45f5b7ef4839b16f9f06cac2d744e34b5b09c1a0d325150d401b87dad108644a/model.onnx", "outputs": "observations/65558486cf0b696980f0cd378b843f3a399f7a048dc28f38eb45156bd962b8cd/outputs.npz" }, "candidate_group": "336a655e64e5fec07886628d7ae79875e9491e382d785ad205a401aaba305c5a", "case_id": "45f5b7ef4839b16f9f06cac2d744e34b5b09c1a0d325150d401b87dad108644a", "category": "metrics", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 5.960464477539063e-08, "max_budget_ratio": 0.005418631376608828, "max_rel": 5.960497517959426e-08, "max_ulp": 1, "mismatched": 0, "name": "cosine_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "reference", "name": "cosine_range", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 1 ], "backend": "reference", "elements": 8, "expected_dtype": "float32", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 1.7881393432617188e-07, "max_budget_ratio": 0.016255894129826483, "max_rel": 1.7881492553878278e-07, "max_ulp": 3, "mismatched": 0, "name": "cosine_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "cosine_range", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 1.1920928955078125e-07, "max_budget_ratio": 0.0108372645149119, "max_rel": 1.1920997167571073e-07, "max_ulp": 2, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 1.1920928955078125e-07, "max_budget_ratio": 0.0108372645149119, "max_rel": 1.1920997167571073e-07, "max_ulp": 2, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 1 ], "backend": "ort", "elements": 8, "expected_dtype": "float32", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } } ], "comparison": { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 1.1920928955078125e-07, "max_budget_ratio": 0.010837263340449007, "max_rel": 1.1920995746469508e-07, "max_ulp": 2, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, "comparison_key": "8ff03a581dd8bf5b3bc162499b2b9995e2df049494f65c766f8d592bbd284511", "contract": "cosine", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "target": { "dtype": "float32", "shape": [ 4, 17 ] }, "x": { "dtype": "float32", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "CosineSimilarity/constant/float32/4x17/seed-20260906", "observation_id": "65558486cf0b696980f0cd378b843f3a399f7a048dc28f38eb45156bd962b8cd", "operation": "CosineSimilarity", "operators": [ "Div", "Max", "Mul", "ReduceSum", "Sqrt" ], "opset": 19, "optimization": "disabled", "output_hash": "cfa02269f64315d4449acc84754c860abd238861d379a59498fc09dfb4e58411", "outputs": {}, "parameters": { "epsilon": 1e-06 }, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.005667, "p95_ms": 0.006716, "repeats": 7, "samples_ms": [ 0.006791, 0.006541, 0.005625, 0.005541, 0.005625, 0.005667, 0.00575 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.061375, "p95_ms": 0.0627917, "repeats": 7, "samples_ms": [ 0.063167, 0.061916, 0.0615, 0.061084, 0.060792, 0.061375, 0.06075 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-06, "rtol": 1e-05 }, "variant": "constant", "warnings": [] }, { "artifacts": { "bundle": "observations/405021199b6febe4c96c84002b91ba0d30a6959f7d1dcd2d31a2594af59994ab/reproducer.zip", "bundle_sha256": "8b12a5079ffb3d1c779cd29230a3ded44c0d2bb2a95bdc27545db84561ac42c2", "case": "artifacts/7db86800973f4b8f32fe2baa93b16946e7366f7d2c3690130a25e6ecdff89b2d/case.json", "inputs": "artifacts/7db86800973f4b8f32fe2baa93b16946e7366f7d2c3690130a25e6ecdff89b2d/inputs.npz", "manifest": "artifacts/7db86800973f4b8f32fe2baa93b16946e7366f7d2c3690130a25e6ecdff89b2d/manifest.json", "model": "artifacts/7db86800973f4b8f32fe2baa93b16946e7366f7d2c3690130a25e6ecdff89b2d/model.onnx", "outputs": "observations/405021199b6febe4c96c84002b91ba0d30a6959f7d1dcd2d31a2594af59994ab/outputs.npz" }, "candidate_group": "4d7cd24a8415ca55bc44260d2d17b382eb9d005b0d9b192fac0c202e50f087c2", "case_id": "7db86800973f4b8f32fe2baa93b16946e7366f7d2c3690130a25e6ecdff89b2d", "category": "metrics", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "cosine_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "backend": "reference", "name": "cosine_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 8, 1 ], "backend": "reference", "elements": 8, "expected_dtype": "float64", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "cosine_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "backend": "ort", "name": "cosine_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 2.220446049250313e-16, "max_budget_ratio": 0.0002018595004534637, "max_rel": 2.2204553505654214e-16, "max_ulp": 2, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 2.220446049250313e-16, "max_budget_ratio": 0.0002018595004534637, "max_rel": 2.2204553505654214e-16, "max_ulp": 2, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 8, 1 ], "backend": "ort", "elements": 8, "expected_dtype": "float64", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } } ], "comparison": { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, "comparison_key": "b66332f945cab287b531deedc3118a6aa7be99e5a7018e8588bfe27f0d3b58d1", "contract": "cosine", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "target": { "dtype": "float64", "shape": [ 4, 17 ] }, "x": { "dtype": "float64", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "CosineSimilarity/constant/float64/4x17/seed-20260906", "observation_id": "405021199b6febe4c96c84002b91ba0d30a6959f7d1dcd2d31a2594af59994ab", "operation": "CosineSimilarity", "operators": [ "Div", "Max", "Mul", "ReduceSum", "Sqrt" ], "opset": 19, "optimization": "all", "output_hash": "be49b6bdc144978837779ef21d0886ebf3c675aadb4b7e1b6841f07ac4b32419", "outputs": {}, "parameters": { "epsilon": 1e-06 }, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.005792, "p95_ms": 0.0062251, "repeats": 7, "samples_ms": [ 0.006167, 0.00625, 0.005834, 0.005792, 0.0055, 0.00525, 0.00525 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.062625, "p95_ms": 0.0639961, "repeats": 7, "samples_ms": [ 0.064084, 0.0635, 0.063791, 0.062083, 0.062625, 0.062, 0.062625 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-13, "rtol": 1e-12 }, "variant": "constant", "warnings": [] }, { "artifacts": { "bundle": "observations/2a8872cfc76516ad78f979a0efd3d54401f9b0b61ac9e52bfe4bfe7ae300665d/reproducer.zip", "bundle_sha256": "f71b28148af8a806291cb0754eeb535e370f1dad8806a7614afc26f1543457ed", "case": "artifacts/7db86800973f4b8f32fe2baa93b16946e7366f7d2c3690130a25e6ecdff89b2d/case.json", "inputs": "artifacts/7db86800973f4b8f32fe2baa93b16946e7366f7d2c3690130a25e6ecdff89b2d/inputs.npz", "manifest": "artifacts/7db86800973f4b8f32fe2baa93b16946e7366f7d2c3690130a25e6ecdff89b2d/manifest.json", "model": "artifacts/7db86800973f4b8f32fe2baa93b16946e7366f7d2c3690130a25e6ecdff89b2d/model.onnx", "outputs": "observations/2a8872cfc76516ad78f979a0efd3d54401f9b0b61ac9e52bfe4bfe7ae300665d/outputs.npz" }, "candidate_group": "4d7cd24a8415ca55bc44260d2d17b382eb9d005b0d9b192fac0c202e50f087c2", "case_id": "7db86800973f4b8f32fe2baa93b16946e7366f7d2c3690130a25e6ecdff89b2d", "category": "metrics", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "cosine_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "backend": "reference", "name": "cosine_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 8, 1 ], "backend": "reference", "elements": 8, "expected_dtype": "float64", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "cosine_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "backend": "ort", "name": "cosine_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 2.220446049250313e-16, "max_budget_ratio": 0.0002018595004534637, "max_rel": 2.2204553505654214e-16, "max_ulp": 2, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 2.220446049250313e-16, "max_budget_ratio": 0.0002018595004534637, "max_rel": 2.2204553505654214e-16, "max_ulp": 2, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 8, 1 ], "backend": "ort", "elements": 8, "expected_dtype": "float64", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } } ], "comparison": { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, "comparison_key": "2a9aed63ab984e3a0ba47ea33a09bfb80c1e892085c631e07abc320afbb7bade", "contract": "cosine", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "target": { "dtype": "float64", "shape": [ 4, 17 ] }, "x": { "dtype": "float64", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "CosineSimilarity/constant/float64/4x17/seed-20260906", "observation_id": "2a8872cfc76516ad78f979a0efd3d54401f9b0b61ac9e52bfe4bfe7ae300665d", "operation": "CosineSimilarity", "operators": [ "Div", "Max", "Mul", "ReduceSum", "Sqrt" ], "opset": 19, "optimization": "disabled", "output_hash": "be49b6bdc144978837779ef21d0886ebf3c675aadb4b7e1b6841f07ac4b32419", "outputs": {}, "parameters": { "epsilon": 1e-06 }, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.00575, "p95_ms": 0.0065704000000000005, "repeats": 7, "samples_ms": [ 0.006541, 0.006583, 0.006083, 0.00575, 0.005583, 0.00525, 0.00525 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.060625, "p95_ms": 0.0615417, "repeats": 7, "samples_ms": [ 0.061542, 0.061291, 0.060042, 0.059917, 0.060625, 0.061541, 0.059917 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-13, "rtol": 1e-12 }, "variant": "constant", "warnings": [] }, { "artifacts": { "bundle": "observations/4b9b8220b7881adb95f9b8acf63bd3e4ae97efa96a07c9aeb19f42357cff6e65/reproducer.zip", "bundle_sha256": "196c24c0d60b0486ead70cec10995bfc3065dd2c4f6cc6ab4a1b76d94357031e", "case": "artifacts/b4eaa370b87ca85ed3d66eba0000d9143683243a97e7a9a0df27405dc2d81f93/case.json", "inputs": "artifacts/b4eaa370b87ca85ed3d66eba0000d9143683243a97e7a9a0df27405dc2d81f93/inputs.npz", "manifest": "artifacts/b4eaa370b87ca85ed3d66eba0000d9143683243a97e7a9a0df27405dc2d81f93/manifest.json", "model": "artifacts/b4eaa370b87ca85ed3d66eba0000d9143683243a97e7a9a0df27405dc2d81f93/model.onnx", "outputs": "observations/4b9b8220b7881adb95f9b8acf63bd3e4ae97efa96a07c9aeb19f42357cff6e65/outputs.npz" }, "candidate_group": "4184796a4574c78172c233d4a7564ef3013922f09589e9be3ec87ef5bb97785c", "case_id": "b4eaa370b87ca85ed3d66eba0000d9143683243a97e7a9a0df27405dc2d81f93", "category": "metrics", "checks": [ { "backend": "reference", "name": "finite_output", "passed": false }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": [ 0, 0 ], "max_abs": null, "max_budget_ratio": null, "max_rel": null, "max_ulp": null, "mismatched": 4, "name": "cosine_float64_oracle", "nonfinite": 4, "numerical_divergence": true, "passed": false, "reason": "nonfinite_output", "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "backend": "reference", "name": "cosine_range", "passed": false }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": [ 0, 0 ], "max_abs": null, "max_budget_ratio": null, "max_rel": null, "max_ulp": null, "mismatched": 4, "name": "batch_partition", "nonfinite": 4, "numerical_divergence": false, "passed": false, "reason": "nonfinite_output", "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": [ 0, 0 ], "max_abs": null, "max_budget_ratio": null, "max_rel": null, "max_ulp": null, "mismatched": 4, "name": "batch_permutation", "nonfinite": 4, "numerical_divergence": false, "passed": false, "reason": "nonfinite_output", "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": [ 0, 0 ], "max_abs": null, "max_budget_ratio": null, "max_rel": null, "max_ulp": null, "mismatched": 4, "name": "batch_reshape", "nonfinite": 4, "numerical_divergence": false, "passed": false, "reason": "nonfinite_output", "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 8, 1 ], "backend": "reference", "elements": 8, "expected_dtype": "float16", "expected_shape": [ 8, 1 ], "first_mismatch": [ 0, 0 ], "max_abs": null, "max_budget_ratio": null, "max_rel": null, "max_ulp": null, "mismatched": 8, "name": "batch_duplicate", "nonfinite": 8, "numerical_divergence": false, "passed": false, "reason": "nonfinite_output", "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "cosine_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "backend": "ort", "name": "cosine_range", "passed": true }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 8, 1 ], "backend": "ort", "elements": 8, "expected_dtype": "float16", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } } ], "comparison": { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": [ 0, 0 ], "max_abs": null, "max_budget_ratio": null, "max_rel": null, "max_ulp": null, "mismatched": 4, "nonfinite": 4, "numerical_divergence": true, "passed": false, "reason": "nonfinite_output", "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, "comparison_key": "fb2f1460d7005ae4ec57835bf6be3d4ae51bb7653a5c2e434ce0e5fccbf295b3", "contract": "cosine", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "target": { "dtype": "float16", "shape": [ 4, 17 ] }, "x": { "dtype": "float16", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "CosineSimilarity/large_offset/float16/4x17/seed-20260906", "observation_id": "4b9b8220b7881adb95f9b8acf63bd3e4ae97efa96a07c9aeb19f42357cff6e65", "operation": "CosineSimilarity", "operators": [ "Div", "Max", "Mul", "ReduceSum", "Sqrt" ], "opset": 19, "optimization": "all", "output_hash": "02dc0188655714aa39e29ab697ff7515e9ac376cabd0f7533b7e7d50f60d3fbf", "outputs": {}, "parameters": { "epsilon": 1e-06 }, "regression": "not_assessed", "seed": 20260906, "status": "divergence", "timings": { "ort": { "median_ms": 0.006959, "p95_ms": 0.007349799999999999, "repeats": 7, "samples_ms": [ 0.007041, 0.007375, 0.006959, 0.007291, 0.006541, 0.0065, 0.006584 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.065334, "p95_ms": 0.0672499, "repeats": 7, "samples_ms": [ 0.06775, 0.066083, 0.065333, 0.065291, 0.06575, 0.065334, 0.065125 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 0.0005, "rtol": 0.005 }, "variant": "large_offset", "warnings": [ { "backend": "reference", "message": "invalid value encountered in divide" }, { "backend": "reference", "message": "overflow encountered in reduce" } ] }, { "artifacts": { "bundle": "observations/5c99cf17923b409ff6684d06115ce5744353d9ca43992ff8bf2f0f4574a15eff/reproducer.zip", "bundle_sha256": "d3cefc51bf244d997cbef9eb4765330e767166a9b5a181ed736b55efc46be910", "case": "artifacts/b4eaa370b87ca85ed3d66eba0000d9143683243a97e7a9a0df27405dc2d81f93/case.json", "inputs": "artifacts/b4eaa370b87ca85ed3d66eba0000d9143683243a97e7a9a0df27405dc2d81f93/inputs.npz", "manifest": "artifacts/b4eaa370b87ca85ed3d66eba0000d9143683243a97e7a9a0df27405dc2d81f93/manifest.json", "model": "artifacts/b4eaa370b87ca85ed3d66eba0000d9143683243a97e7a9a0df27405dc2d81f93/model.onnx", "outputs": "observations/5c99cf17923b409ff6684d06115ce5744353d9ca43992ff8bf2f0f4574a15eff/outputs.npz" }, "candidate_group": "4184796a4574c78172c233d4a7564ef3013922f09589e9be3ec87ef5bb97785c", "case_id": "b4eaa370b87ca85ed3d66eba0000d9143683243a97e7a9a0df27405dc2d81f93", "category": "metrics", "checks": [ { "backend": "reference", "name": "finite_output", "passed": false }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": [ 0, 0 ], "max_abs": null, "max_budget_ratio": null, "max_rel": null, "max_ulp": null, "mismatched": 4, "name": "cosine_float64_oracle", "nonfinite": 4, "numerical_divergence": true, "passed": false, "reason": "nonfinite_output", "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "backend": "reference", "name": "cosine_range", "passed": false }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": [ 0, 0 ], "max_abs": null, "max_budget_ratio": null, "max_rel": null, "max_ulp": null, "mismatched": 4, "name": "batch_partition", "nonfinite": 4, "numerical_divergence": false, "passed": false, "reason": "nonfinite_output", "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": [ 0, 0 ], "max_abs": null, "max_budget_ratio": null, "max_rel": null, "max_ulp": null, "mismatched": 4, "name": "batch_permutation", "nonfinite": 4, "numerical_divergence": false, "passed": false, "reason": "nonfinite_output", "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": [ 0, 0 ], "max_abs": null, "max_budget_ratio": null, "max_rel": null, "max_ulp": null, "mismatched": 4, "name": "batch_reshape", "nonfinite": 4, "numerical_divergence": false, "passed": false, "reason": "nonfinite_output", "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 8, 1 ], "backend": "reference", "elements": 8, "expected_dtype": "float16", "expected_shape": [ 8, 1 ], "first_mismatch": [ 0, 0 ], "max_abs": null, "max_budget_ratio": null, "max_rel": null, "max_ulp": null, "mismatched": 8, "name": "batch_duplicate", "nonfinite": 8, "numerical_divergence": false, "passed": false, "reason": "nonfinite_output", "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "cosine_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "backend": "ort", "name": "cosine_range", "passed": true }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 8, 1 ], "backend": "ort", "elements": 8, "expected_dtype": "float16", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } } ], "comparison": { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": [ 0, 0 ], "max_abs": null, "max_budget_ratio": null, "max_rel": null, "max_ulp": null, "mismatched": 4, "nonfinite": 4, "numerical_divergence": true, "passed": false, "reason": "nonfinite_output", "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, "comparison_key": "6ac42c5510e83483315c52678656b501aae98aa8166b08031dcc7e66a83590b4", "contract": "cosine", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "target": { "dtype": "float16", "shape": [ 4, 17 ] }, "x": { "dtype": "float16", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "CosineSimilarity/large_offset/float16/4x17/seed-20260906", "observation_id": "5c99cf17923b409ff6684d06115ce5744353d9ca43992ff8bf2f0f4574a15eff", "operation": "CosineSimilarity", "operators": [ "Div", "Max", "Mul", "ReduceSum", "Sqrt" ], "opset": 19, "optimization": "disabled", "output_hash": "02dc0188655714aa39e29ab697ff7515e9ac376cabd0f7533b7e7d50f60d3fbf", "outputs": {}, "parameters": { "epsilon": 1e-06 }, "regression": "not_assessed", "seed": 20260906, "status": "divergence", "timings": { "ort": { "median_ms": 0.00675, "p95_ms": 0.0075, "repeats": 7, "samples_ms": [ 0.0075, 0.0075, 0.007125, 0.00675, 0.006542, 0.0065, 0.006541 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.075291, "p95_ms": 0.13203739999999994, "repeats": 7, "samples_ms": [ 0.0755, 0.156125, 0.075833, 0.070792, 0.067917, 0.06725, 0.075291 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 0.0005, "rtol": 0.005 }, "variant": "large_offset", "warnings": [ { "backend": "reference", "message": "invalid value encountered in divide" }, { "backend": "reference", "message": "overflow encountered in reduce" } ] }, { "artifacts": { "bundle": "observations/d21ab7802782f2e1e951d9605d0c0da645bc12c6cf75fc27efdbc45e1aabcb46/reproducer.zip", "bundle_sha256": "d51ad7d746d5e8ad6316bcf912b4860ebbeb58a5ebe7ec7671edbdb015b17c54", "case": "artifacts/79bb0818a9b5e5b9253188e80d99d16617261e3951e0a14a4a172034811859cd/case.json", "inputs": "artifacts/79bb0818a9b5e5b9253188e80d99d16617261e3951e0a14a4a172034811859cd/inputs.npz", "manifest": "artifacts/79bb0818a9b5e5b9253188e80d99d16617261e3951e0a14a4a172034811859cd/manifest.json", "model": "artifacts/79bb0818a9b5e5b9253188e80d99d16617261e3951e0a14a4a172034811859cd/model.onnx", "outputs": "observations/d21ab7802782f2e1e951d9605d0c0da645bc12c6cf75fc27efdbc45e1aabcb46/outputs.npz" }, "candidate_group": "bee5fa7d6bb3fbcf1ad223959a356c0940d110dacc84be7d113ae5169655cebc", "case_id": "79bb0818a9b5e5b9253188e80d99d16617261e3951e0a14a4a172034811859cd", "category": "metrics", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 1.1920928955078125e-07, "max_budget_ratio": 0.010837208140980112, "max_rel": 1.1920928955078125e-07, "max_ulp": 1, "mismatched": 0, "name": "cosine_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "reference", "name": "cosine_range", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 1 ], "backend": "reference", "elements": 8, "expected_dtype": "float32", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 1.1920928955078125e-07, "max_budget_ratio": 0.010837208140980112, "max_rel": 1.1920928955078125e-07, "max_ulp": 1, "mismatched": 0, "name": "cosine_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "cosine_range", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 1.1920928955078125e-07, "max_budget_ratio": 0.010837206966529437, "max_rel": 1.1920927533992823e-07, "max_ulp": 1, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 1.1920928955078125e-07, "max_budget_ratio": 0.010837208140980112, "max_rel": 1.1920928955078125e-07, "max_ulp": 1, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 1 ], "backend": "ort", "elements": 8, "expected_dtype": "float32", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } } ], "comparison": { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 1.1920928955078125e-07, "max_budget_ratio": 0.010837208140980112, "max_rel": 1.1920928955078125e-07, "max_ulp": 1, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, "comparison_key": "74ba94c775e9c1f756d26e493c962bc106311c9bf2fc578cb91f391c81f300ea", "contract": "cosine", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "target": { "dtype": "float32", "shape": [ 4, 17 ] }, "x": { "dtype": "float32", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "CosineSimilarity/large_offset/float32/4x17/seed-20260906", "observation_id": "d21ab7802782f2e1e951d9605d0c0da645bc12c6cf75fc27efdbc45e1aabcb46", "operation": "CosineSimilarity", "operators": [ "Div", "Max", "Mul", "ReduceSum", "Sqrt" ], "opset": 19, "optimization": "all", "output_hash": "f5c9f5c45416ab8c12e1cf7c8b513370566cc4a71d235d607083002ad82ae3b8", "outputs": {}, "parameters": { "epsilon": 1e-06 }, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.006166, "p95_ms": 0.0070293, "repeats": 7, "samples_ms": [ 0.007167, 0.006708, 0.006333, 0.006166, 0.006084, 0.005833, 0.005833 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.061084, "p95_ms": 0.0618707, "repeats": 7, "samples_ms": [ 0.061958, 0.061041, 0.060625, 0.061667, 0.061209, 0.061084, 0.060375 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-06, "rtol": 1e-05 }, "variant": "large_offset", "warnings": [] }, { "artifacts": { "bundle": "observations/e8e639e38853be7732da24edacc68b1976b19b28820a4eeffe82916d832867e4/reproducer.zip", "bundle_sha256": "405e08a2f0e59fc6323d53d39b0c75de99d92d7e3722275d30a6cdfef3b9a000", "case": "artifacts/79bb0818a9b5e5b9253188e80d99d16617261e3951e0a14a4a172034811859cd/case.json", "inputs": "artifacts/79bb0818a9b5e5b9253188e80d99d16617261e3951e0a14a4a172034811859cd/inputs.npz", "manifest": "artifacts/79bb0818a9b5e5b9253188e80d99d16617261e3951e0a14a4a172034811859cd/manifest.json", "model": "artifacts/79bb0818a9b5e5b9253188e80d99d16617261e3951e0a14a4a172034811859cd/model.onnx", "outputs": "observations/e8e639e38853be7732da24edacc68b1976b19b28820a4eeffe82916d832867e4/outputs.npz" }, "candidate_group": "bee5fa7d6bb3fbcf1ad223959a356c0940d110dacc84be7d113ae5169655cebc", "case_id": "79bb0818a9b5e5b9253188e80d99d16617261e3951e0a14a4a172034811859cd", "category": "metrics", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 1.1920928955078125e-07, "max_budget_ratio": 0.010837208140980112, "max_rel": 1.1920928955078125e-07, "max_ulp": 1, "mismatched": 0, "name": "cosine_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "reference", "name": "cosine_range", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 1 ], "backend": "reference", "elements": 8, "expected_dtype": "float32", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 1.1920928955078125e-07, "max_budget_ratio": 0.010837208140980112, "max_rel": 1.1920928955078125e-07, "max_ulp": 1, "mismatched": 0, "name": "cosine_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "cosine_range", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 1.1920928955078125e-07, "max_budget_ratio": 0.010837206966529437, "max_rel": 1.1920927533992823e-07, "max_ulp": 1, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 1.1920928955078125e-07, "max_budget_ratio": 0.010837208140980112, "max_rel": 1.1920928955078125e-07, "max_ulp": 1, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 1 ], "backend": "ort", "elements": 8, "expected_dtype": "float32", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } } ], "comparison": { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 1.1920928955078125e-07, "max_budget_ratio": 0.010837208140980112, "max_rel": 1.1920928955078125e-07, "max_ulp": 1, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, "comparison_key": "0cdf426cd0a276f4ddfa3f6a52f77bc216cc70ce7a5e6396beb1bcf532dbfec3", "contract": "cosine", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "target": { "dtype": "float32", "shape": [ 4, 17 ] }, "x": { "dtype": "float32", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "CosineSimilarity/large_offset/float32/4x17/seed-20260906", "observation_id": "e8e639e38853be7732da24edacc68b1976b19b28820a4eeffe82916d832867e4", "operation": "CosineSimilarity", "operators": [ "Div", "Max", "Mul", "ReduceSum", "Sqrt" ], "opset": 19, "optimization": "disabled", "output_hash": "f5c9f5c45416ab8c12e1cf7c8b513370566cc4a71d235d607083002ad82ae3b8", "outputs": {}, "parameters": { "epsilon": 1e-06 }, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.005208, "p95_ms": 0.011550099999999995, "repeats": 7, "samples_ms": [ 0.01375, 0.006417, 0.0055, 0.005208, 0.005208, 0.005, 0.005167 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.062708, "p95_ms": 0.0633545, "repeats": 7, "samples_ms": [ 0.063542, 0.062708, 0.06275, 0.06225, 0.06175, 0.062917, 0.061417 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-06, "rtol": 1e-05 }, "variant": "large_offset", "warnings": [] }, { "artifacts": { "bundle": "observations/b430d85678102d9cbc9382f6877b06227869ca55c565c727d6efe07f955ac1c8/reproducer.zip", "bundle_sha256": "97a551262a93b6fc8e94397db70d86b8ce06dc61328e0995d881166ea91739d3", "case": "artifacts/65650fb3400f4a4c4394d7f4a5659a08ddf82de3571f9ecbf9a01d96391a2322/case.json", "inputs": "artifacts/65650fb3400f4a4c4394d7f4a5659a08ddf82de3571f9ecbf9a01d96391a2322/inputs.npz", "manifest": "artifacts/65650fb3400f4a4c4394d7f4a5659a08ddf82de3571f9ecbf9a01d96391a2322/manifest.json", "model": "artifacts/65650fb3400f4a4c4394d7f4a5659a08ddf82de3571f9ecbf9a01d96391a2322/model.onnx", "outputs": "observations/b430d85678102d9cbc9382f6877b06227869ca55c565c727d6efe07f955ac1c8/outputs.npz" }, "candidate_group": "291cfac8af93efa86ccda231e5477cd1c820c361152a481a9f9eeb5461498c4a", "case_id": "65650fb3400f4a4c4394d7f4a5659a08ddf82de3571f9ecbf9a01d96391a2322", "category": "metrics", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "cosine_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "backend": "reference", "name": "cosine_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 8, 1 ], "backend": "reference", "elements": 8, "expected_dtype": "float64", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "cosine_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "backend": "ort", "name": "cosine_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 2.220446049250313e-16, "max_budget_ratio": 0.00020185873175009576, "max_rel": 2.2204460492511276e-16, "max_ulp": 2, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 2.220446049250313e-16, "max_budget_ratio": 0.00020185873175009576, "max_rel": 2.2204460492511276e-16, "max_ulp": 2, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 8, 1 ], "backend": "ort", "elements": 8, "expected_dtype": "float64", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } } ], "comparison": { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, "comparison_key": "1dd0ce4f057cf03047176afefd1fe94ce2754eebff6ae771bb81505e467e98e4", "contract": "cosine", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "target": { "dtype": "float64", "shape": [ 4, 17 ] }, "x": { "dtype": "float64", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "CosineSimilarity/large_offset/float64/4x17/seed-20260906", "observation_id": "b430d85678102d9cbc9382f6877b06227869ca55c565c727d6efe07f955ac1c8", "operation": "CosineSimilarity", "operators": [ "Div", "Max", "Mul", "ReduceSum", "Sqrt" ], "opset": 19, "optimization": "all", "output_hash": "b9bcf3b7841350d8548436ebe92ee1d82b11c7f120d5e61ecd531beb0fbfd8f5", "outputs": {}, "parameters": { "epsilon": 1e-06 }, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.005834, "p95_ms": 0.0067037, "repeats": 7, "samples_ms": [ 0.006791, 0.0065, 0.005916, 0.005834, 0.005666, 0.005167, 0.005333 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.061667, "p95_ms": 0.0678458, "repeats": 7, "samples_ms": [ 0.062375, 0.068333, 0.061667, 0.066709, 0.061542, 0.060542, 0.060875 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-13, "rtol": 1e-12 }, "variant": "large_offset", "warnings": [] }, { "artifacts": { "bundle": "observations/99dd374ed7203fd34a75cbc4d31a3119fa45ed804753947f4890244032defb37/reproducer.zip", "bundle_sha256": "063aadd2b57549aafee5d55455160c482e98453c365fadd34b7ba040d9d011fb", "case": "artifacts/65650fb3400f4a4c4394d7f4a5659a08ddf82de3571f9ecbf9a01d96391a2322/case.json", "inputs": "artifacts/65650fb3400f4a4c4394d7f4a5659a08ddf82de3571f9ecbf9a01d96391a2322/inputs.npz", "manifest": "artifacts/65650fb3400f4a4c4394d7f4a5659a08ddf82de3571f9ecbf9a01d96391a2322/manifest.json", "model": "artifacts/65650fb3400f4a4c4394d7f4a5659a08ddf82de3571f9ecbf9a01d96391a2322/model.onnx", "outputs": "observations/99dd374ed7203fd34a75cbc4d31a3119fa45ed804753947f4890244032defb37/outputs.npz" }, "candidate_group": "291cfac8af93efa86ccda231e5477cd1c820c361152a481a9f9eeb5461498c4a", "case_id": "65650fb3400f4a4c4394d7f4a5659a08ddf82de3571f9ecbf9a01d96391a2322", "category": "metrics", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "cosine_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "backend": "reference", "name": "cosine_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 8, 1 ], "backend": "reference", "elements": 8, "expected_dtype": "float64", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "cosine_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "backend": "ort", "name": "cosine_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 2.220446049250313e-16, "max_budget_ratio": 0.00020185873175009576, "max_rel": 2.2204460492511276e-16, "max_ulp": 2, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 2.220446049250313e-16, "max_budget_ratio": 0.00020185873175009576, "max_rel": 2.2204460492511276e-16, "max_ulp": 2, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 8, 1 ], "backend": "ort", "elements": 8, "expected_dtype": "float64", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } } ], "comparison": { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, "comparison_key": "8ddb1a3c0d6c0feb3397b7d077b8a1fef5d239aa09d4b2c550ecb4325e308bd1", "contract": "cosine", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "target": { "dtype": "float64", "shape": [ 4, 17 ] }, "x": { "dtype": "float64", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "CosineSimilarity/large_offset/float64/4x17/seed-20260906", "observation_id": "99dd374ed7203fd34a75cbc4d31a3119fa45ed804753947f4890244032defb37", "operation": "CosineSimilarity", "operators": [ "Div", "Max", "Mul", "ReduceSum", "Sqrt" ], "opset": 19, "optimization": "disabled", "output_hash": "b9bcf3b7841350d8548436ebe92ee1d82b11c7f120d5e61ecd531beb0fbfd8f5", "outputs": {}, "parameters": { "epsilon": 1e-06 }, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.006125, "p95_ms": 0.006758, "repeats": 7, "samples_ms": [ 0.006583, 0.006833, 0.00625, 0.006125, 0.005917, 0.006, 0.005834 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.062708, "p95_ms": 0.0718417, "repeats": 7, "samples_ms": [ 0.064708, 0.062458, 0.068166, 0.061916, 0.073417, 0.062708, 0.061416 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-13, "rtol": 1e-12 }, "variant": "large_offset", "warnings": [] }, { "artifacts": { "bundle": "observations/6979e9101da47bd0cba2d697b40271acf3dfa360cb8836e15b539b0fecf378d4/reproducer.zip", "bundle_sha256": "473829d7a7e6afd0bba80c3f9c907987da4cc7623d5537cd58784d3cec824232", "case": "artifacts/c970752626d7817a7e32844a90c57bacab80e91bd0556638d1fcb8c65e20678b/case.json", "inputs": "artifacts/c970752626d7817a7e32844a90c57bacab80e91bd0556638d1fcb8c65e20678b/inputs.npz", "manifest": "artifacts/c970752626d7817a7e32844a90c57bacab80e91bd0556638d1fcb8c65e20678b/manifest.json", "model": "artifacts/c970752626d7817a7e32844a90c57bacab80e91bd0556638d1fcb8c65e20678b/model.onnx", "outputs": "observations/6979e9101da47bd0cba2d697b40271acf3dfa360cb8836e15b539b0fecf378d4/outputs.npz" }, "candidate_group": "bc3d4c19e01a334147deea1459ea1893543cba6f2bfb07c120ae9e9e008bf65d", "case_id": "c970752626d7817a7e32844a90c57bacab80e91bd0556638d1fcb8c65e20678b", "category": "metrics", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.00048828125, "max_budget_ratio": 0.0887784090909091, "max_rel": 0.00048828125, "max_ulp": 1, "mismatched": 0, "name": "cosine_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "backend": "reference", "name": "cosine_range", "passed": true }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 8, 1 ], "backend": "reference", "elements": 8, "expected_dtype": "float16", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "cosine_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "backend": "ort", "name": "cosine_range", "passed": true }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 8, 1 ], "backend": "ort", "elements": 8, "expected_dtype": "float16", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } } ], "comparison": { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.00048828125, "max_budget_ratio": 0.08881783462119193, "max_rel": 0.0004885197850512946, "max_ulp": 1, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, "comparison_key": "b22f5338ade10dbb80f93582eac9de39de0df312a38c599c854c2c7232ae5e07", "contract": "cosine", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "target": { "dtype": "float16", "shape": [ 4, 17 ] }, "x": { "dtype": "float16", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "CosineSimilarity/ordinary/float16/4x17/seed-20260906", "observation_id": "6979e9101da47bd0cba2d697b40271acf3dfa360cb8836e15b539b0fecf378d4", "operation": "CosineSimilarity", "operators": [ "Div", "Max", "Mul", "ReduceSum", "Sqrt" ], "opset": 19, "optimization": "all", "output_hash": "7969ee125957deeb7efd781549d7be2039d99ae6a4c776247bd51dd75afeace2", "outputs": {}, "parameters": { "epsilon": 1e-06 }, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.00725, "p95_ms": 0.0077706, "repeats": 7, "samples_ms": [ 0.007625, 0.007833, 0.007584, 0.00725, 0.007125, 0.0065, 0.006541 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.06475, "p95_ms": 0.065933, "repeats": 7, "samples_ms": [ 0.066083, 0.065334, 0.064042, 0.063958, 0.06475, 0.065583, 0.064583 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 0.0005, "rtol": 0.005 }, "variant": "ordinary", "warnings": [ { "backend": "reference", "message": "underflow encountered in multiply" } ] }, { "artifacts": { "bundle": "observations/b6d9bd2233cedf740effc7b7e3c5ca341231f888cae0bfe4432c361ab6e09b0d/reproducer.zip", "bundle_sha256": "5031a7fc6adbf45a536db81ca75f7182d513af76159db567d2124627ebd285f5", "case": "artifacts/c970752626d7817a7e32844a90c57bacab80e91bd0556638d1fcb8c65e20678b/case.json", "inputs": "artifacts/c970752626d7817a7e32844a90c57bacab80e91bd0556638d1fcb8c65e20678b/inputs.npz", "manifest": "artifacts/c970752626d7817a7e32844a90c57bacab80e91bd0556638d1fcb8c65e20678b/manifest.json", "model": "artifacts/c970752626d7817a7e32844a90c57bacab80e91bd0556638d1fcb8c65e20678b/model.onnx", "outputs": "observations/b6d9bd2233cedf740effc7b7e3c5ca341231f888cae0bfe4432c361ab6e09b0d/outputs.npz" }, "candidate_group": "bc3d4c19e01a334147deea1459ea1893543cba6f2bfb07c120ae9e9e008bf65d", "case_id": "c970752626d7817a7e32844a90c57bacab80e91bd0556638d1fcb8c65e20678b", "category": "metrics", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.00048828125, "max_budget_ratio": 0.0887784090909091, "max_rel": 0.00048828125, "max_ulp": 1, "mismatched": 0, "name": "cosine_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "backend": "reference", "name": "cosine_range", "passed": true }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 8, 1 ], "backend": "reference", "elements": 8, "expected_dtype": "float16", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "cosine_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "backend": "ort", "name": "cosine_range", "passed": true }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 8, 1 ], "backend": "ort", "elements": 8, "expected_dtype": "float16", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } } ], "comparison": { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.00048828125, "max_budget_ratio": 0.08881783462119193, "max_rel": 0.0004885197850512946, "max_ulp": 1, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, "comparison_key": "feabbef56dc578a1dc38b9cd75e973f383b15ed9c5b367086aadc163769fb370", "contract": "cosine", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "target": { "dtype": "float16", "shape": [ 4, 17 ] }, "x": { "dtype": "float16", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "CosineSimilarity/ordinary/float16/4x17/seed-20260906", "observation_id": "b6d9bd2233cedf740effc7b7e3c5ca341231f888cae0bfe4432c361ab6e09b0d", "operation": "CosineSimilarity", "operators": [ "Div", "Max", "Mul", "ReduceSum", "Sqrt" ], "opset": 19, "optimization": "disabled", "output_hash": "7969ee125957deeb7efd781549d7be2039d99ae6a4c776247bd51dd75afeace2", "outputs": {}, "parameters": { "epsilon": 1e-06 }, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.007167, "p95_ms": 0.007791, "repeats": 7, "samples_ms": [ 0.007791, 0.0075, 0.007, 0.007167, 0.006834, 0.006625, 0.007791 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.067084, "p95_ms": 0.09432949999999997, "repeats": 7, "samples_ms": [ 0.103667, 0.072542, 0.068291, 0.067084, 0.065291, 0.065084, 0.066541 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 0.0005, "rtol": 0.005 }, "variant": "ordinary", "warnings": [ { "backend": "reference", "message": "underflow encountered in multiply" } ] }, { "artifacts": { "bundle": "observations/e11bd5ba03b63e747c246437784cd330d88b0c17cc5b08c3dfa6e925ffd27992/reproducer.zip", "bundle_sha256": "c88b8b2c38aeb63213a8337042b3cab3fe92a6aaf74ab1a1d3e54a232d386af1", "case": "artifacts/727e46e4c03d450d851a673814c9d409fdb10e26516f7060f5a4cb37308c4691/case.json", "inputs": "artifacts/727e46e4c03d450d851a673814c9d409fdb10e26516f7060f5a4cb37308c4691/inputs.npz", "manifest": "artifacts/727e46e4c03d450d851a673814c9d409fdb10e26516f7060f5a4cb37308c4691/manifest.json", "model": "artifacts/727e46e4c03d450d851a673814c9d409fdb10e26516f7060f5a4cb37308c4691/model.onnx", "outputs": "observations/e11bd5ba03b63e747c246437784cd330d88b0c17cc5b08c3dfa6e925ffd27992/outputs.npz" }, "candidate_group": "9506b83c5011ae3cd67dfff831f07ead340ad873c9e9f6ee850e063585b2b642", "case_id": "727e46e4c03d450d851a673814c9d409fdb10e26516f7060f5a4cb37308c4691", "category": "metrics", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 1.1920928955078125e-07, "max_budget_ratio": 0.010838069081808973, "max_rel": 1.1921970701757e-07, "max_ulp": 2, "mismatched": 0, "name": "cosine_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "reference", "name": "cosine_range", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 1 ], "backend": "reference", "elements": 8, "expected_dtype": "float32", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 1.1920928955078125e-07, "max_budget_ratio": 0.010838069081808973, "max_rel": 1.1921970701757e-07, "max_ulp": 2, "mismatched": 0, "name": "cosine_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "cosine_range", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 1.1920928955078125e-07, "max_budget_ratio": 0.0108376427052022, "max_rel": 1.192145477989537e-07, "max_ulp": 2, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 1.1920928955078125e-07, "max_budget_ratio": 0.010838070256446515, "max_rel": 1.1921972123091023e-07, "max_ulp": 2, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 1 ], "backend": "ort", "elements": 8, "expected_dtype": "float32", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } } ], "comparison": { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 1.1920928955078125e-07, "max_budget_ratio": 0.010837956905096984, "max_rel": 1.192183496591935e-07, "max_ulp": 2, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, "comparison_key": "27bb06e85ac63a560ab48fc9bcaac15b12904048947d57931261744284d17177", "contract": "cosine", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "target": { "dtype": "float32", "shape": [ 4, 17 ] }, "x": { "dtype": "float32", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "CosineSimilarity/ordinary/float32/4x17/seed-20260906", "observation_id": "e11bd5ba03b63e747c246437784cd330d88b0c17cc5b08c3dfa6e925ffd27992", "operation": "CosineSimilarity", "operators": [ "Div", "Max", "Mul", "ReduceSum", "Sqrt" ], "opset": 19, "optimization": "all", "output_hash": "feb8def4c07a3419da010fe949afc5e9f42af14e2ceed2bd2ee44c40652f1e2d", "outputs": {}, "parameters": { "epsilon": 1e-06 }, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.005792, "p95_ms": 0.0064624999999999995, "repeats": 7, "samples_ms": [ 0.0065, 0.006375, 0.005916, 0.005625, 0.0055, 0.005458, 0.005792 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.060833, "p95_ms": 0.0625, "repeats": 7, "samples_ms": [ 0.062875, 0.061333, 0.060833, 0.060292, 0.060167, 0.061625, 0.060042 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-06, "rtol": 1e-05 }, "variant": "ordinary", "warnings": [] }, { "artifacts": { "bundle": "observations/96547245b28b0559a9eb4e299a0cf6224180396f5a8790ef723fb331621f1d3f/reproducer.zip", "bundle_sha256": "749e01d82758ebd18316a4d50f65c6225bedf03fb3e7dbe81dd5710064b548fc", "case": "artifacts/727e46e4c03d450d851a673814c9d409fdb10e26516f7060f5a4cb37308c4691/case.json", "inputs": "artifacts/727e46e4c03d450d851a673814c9d409fdb10e26516f7060f5a4cb37308c4691/inputs.npz", "manifest": "artifacts/727e46e4c03d450d851a673814c9d409fdb10e26516f7060f5a4cb37308c4691/manifest.json", "model": "artifacts/727e46e4c03d450d851a673814c9d409fdb10e26516f7060f5a4cb37308c4691/model.onnx", "outputs": "observations/96547245b28b0559a9eb4e299a0cf6224180396f5a8790ef723fb331621f1d3f/outputs.npz" }, "candidate_group": "9506b83c5011ae3cd67dfff831f07ead340ad873c9e9f6ee850e063585b2b642", "case_id": "727e46e4c03d450d851a673814c9d409fdb10e26516f7060f5a4cb37308c4691", "category": "metrics", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 1.1920928955078125e-07, "max_budget_ratio": 0.010838069081808973, "max_rel": 1.1921970701757e-07, "max_ulp": 2, "mismatched": 0, "name": "cosine_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "reference", "name": "cosine_range", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 1 ], "backend": "reference", "elements": 8, "expected_dtype": "float32", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 1.1920928955078125e-07, "max_budget_ratio": 0.010838069081808973, "max_rel": 1.1921970701757e-07, "max_ulp": 2, "mismatched": 0, "name": "cosine_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "cosine_range", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 1.1920928955078125e-07, "max_budget_ratio": 0.0108376427052022, "max_rel": 1.192145477989537e-07, "max_ulp": 2, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 1.1920928955078125e-07, "max_budget_ratio": 0.010838070256446515, "max_rel": 1.1921972123091023e-07, "max_ulp": 2, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 1 ], "backend": "ort", "elements": 8, "expected_dtype": "float32", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } } ], "comparison": { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 1.1920928955078125e-07, "max_budget_ratio": 0.010837956905096984, "max_rel": 1.192183496591935e-07, "max_ulp": 2, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, "comparison_key": "db565522f2c772f1c32cc67f8ea66d71293a37d2e90c9cae28d85c361cd1fb63", "contract": "cosine", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "target": { "dtype": "float32", "shape": [ 4, 17 ] }, "x": { "dtype": "float32", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "CosineSimilarity/ordinary/float32/4x17/seed-20260906", "observation_id": "96547245b28b0559a9eb4e299a0cf6224180396f5a8790ef723fb331621f1d3f", "operation": "CosineSimilarity", "operators": [ "Div", "Max", "Mul", "ReduceSum", "Sqrt" ], "opset": 19, "optimization": "disabled", "output_hash": "feb8def4c07a3419da010fe949afc5e9f42af14e2ceed2bd2ee44c40652f1e2d", "outputs": {}, "parameters": { "epsilon": 1e-06 }, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.005416, "p95_ms": 0.0062125, "repeats": 7, "samples_ms": [ 0.00625, 0.006125, 0.00575, 0.005416, 0.005041, 0.005125, 0.00525 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.062709, "p95_ms": 0.08940009999999998, "repeats": 7, "samples_ms": [ 0.06375, 0.062709, 0.062291, 0.061584, 0.060958, 0.065542, 0.099625 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-06, "rtol": 1e-05 }, "variant": "ordinary", "warnings": [] }, { "artifacts": { "bundle": "observations/7c9df42059c89a428c92cc81e60ba4ca5da987daf73bfeac5045ce10e2a6f596/reproducer.zip", "bundle_sha256": "aea3948ff2343bafdb096e0c5799e61d55f4a9e7a363c06c416da5022b4110e7", "case": "artifacts/14b8721b757a112ff08f2a2b0be0ddd75e0a69aec832bbc35cffbe6b23262950/case.json", "inputs": "artifacts/14b8721b757a112ff08f2a2b0be0ddd75e0a69aec832bbc35cffbe6b23262950/inputs.npz", "manifest": "artifacts/14b8721b757a112ff08f2a2b0be0ddd75e0a69aec832bbc35cffbe6b23262950/manifest.json", "model": "artifacts/14b8721b757a112ff08f2a2b0be0ddd75e0a69aec832bbc35cffbe6b23262950/model.onnx", "outputs": "observations/7c9df42059c89a428c92cc81e60ba4ca5da987daf73bfeac5045ce10e2a6f596/outputs.npz" }, "candidate_group": "5dbc7210ee24ca251ee48cd3039ee8d95b7949bff08ae1ff10fc66f774123f36", "case_id": "14b8721b757a112ff08f2a2b0be0ddd75e0a69aec832bbc35cffbe6b23262950", "category": "metrics", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "cosine_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "backend": "reference", "name": "cosine_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 8, 1 ], "backend": "reference", "elements": 8, "expected_dtype": "float64", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 3.3306690738754696e-16, "max_budget_ratio": 0.0003027989751097593, "max_rel": 3.3308006919133703e-16, "max_ulp": 3, "mismatched": 0, "name": "cosine_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "backend": "ort", "name": "cosine_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 3.3306690738754696e-16, "max_budget_ratio": 0.00030279897510975917, "max_rel": 3.3308006919133693e-16, "max_ulp": 3, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 3.3306690738754696e-16, "max_budget_ratio": 0.00030279897510975917, "max_rel": 3.3308006919133693e-16, "max_ulp": 3, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 8, 1 ], "backend": "ort", "elements": 8, "expected_dtype": "float64", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } } ], "comparison": { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 3.3306690738754696e-16, "max_budget_ratio": 0.0003027989751097593, "max_rel": 3.3308006919133703e-16, "max_ulp": 3, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, "comparison_key": "e00966f7765f17ca30b2c5c3d73482a58fdf312706bc80745d87e91df517b203", "contract": "cosine", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "target": { "dtype": "float64", "shape": [ 4, 17 ] }, "x": { "dtype": "float64", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "CosineSimilarity/ordinary/float64/4x17/seed-20260906", "observation_id": "7c9df42059c89a428c92cc81e60ba4ca5da987daf73bfeac5045ce10e2a6f596", "operation": "CosineSimilarity", "operators": [ "Div", "Max", "Mul", "ReduceSum", "Sqrt" ], "opset": 19, "optimization": "all", "output_hash": "ac4a600214113fe129ae23e5b4a7d946a3adedc0f7d91822ddb71fbc54824c5a", "outputs": {}, "parameters": { "epsilon": 1e-06 }, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.0055, "p95_ms": 0.0066456, "repeats": 7, "samples_ms": [ 0.006708, 0.0065, 0.005959, 0.0055, 0.005291, 0.005375, 0.005334 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.060917, "p95_ms": 0.0627956, "repeats": 7, "samples_ms": [ 0.063083, 0.062125, 0.060917, 0.060667, 0.060458, 0.061292, 0.060417 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-13, "rtol": 1e-12 }, "variant": "ordinary", "warnings": [] }, { "artifacts": { "bundle": "observations/c76f9369d81c49e219e237791b4a985bf4bf38720b94062dfea1bc07162652f6/reproducer.zip", "bundle_sha256": "819a8c1e43192d02f9ce454b56a9dbe544cef45ef18fc691e22a1ae0beaa3a8e", "case": "artifacts/14b8721b757a112ff08f2a2b0be0ddd75e0a69aec832bbc35cffbe6b23262950/case.json", "inputs": "artifacts/14b8721b757a112ff08f2a2b0be0ddd75e0a69aec832bbc35cffbe6b23262950/inputs.npz", "manifest": "artifacts/14b8721b757a112ff08f2a2b0be0ddd75e0a69aec832bbc35cffbe6b23262950/manifest.json", "model": "artifacts/14b8721b757a112ff08f2a2b0be0ddd75e0a69aec832bbc35cffbe6b23262950/model.onnx", "outputs": "observations/c76f9369d81c49e219e237791b4a985bf4bf38720b94062dfea1bc07162652f6/outputs.npz" }, "candidate_group": "5dbc7210ee24ca251ee48cd3039ee8d95b7949bff08ae1ff10fc66f774123f36", "case_id": "14b8721b757a112ff08f2a2b0be0ddd75e0a69aec832bbc35cffbe6b23262950", "category": "metrics", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "cosine_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "backend": "reference", "name": "cosine_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 8, 1 ], "backend": "reference", "elements": 8, "expected_dtype": "float64", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 3.3306690738754696e-16, "max_budget_ratio": 0.0003027989751097593, "max_rel": 3.3308006919133703e-16, "max_ulp": 3, "mismatched": 0, "name": "cosine_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "backend": "ort", "name": "cosine_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 3.3306690738754696e-16, "max_budget_ratio": 0.00030279897510975917, "max_rel": 3.3308006919133693e-16, "max_ulp": 3, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 3.3306690738754696e-16, "max_budget_ratio": 0.00030279897510975917, "max_rel": 3.3308006919133693e-16, "max_ulp": 3, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 8, 1 ], "backend": "ort", "elements": 8, "expected_dtype": "float64", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } } ], "comparison": { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 3.3306690738754696e-16, "max_budget_ratio": 0.0003027989751097593, "max_rel": 3.3308006919133703e-16, "max_ulp": 3, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, "comparison_key": "88e94669bcaba3cc133d9791c63f250103480ce3e950b53e4af24aa75996caa1", "contract": "cosine", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "target": { "dtype": "float64", "shape": [ 4, 17 ] }, "x": { "dtype": "float64", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "CosineSimilarity/ordinary/float64/4x17/seed-20260906", "observation_id": "c76f9369d81c49e219e237791b4a985bf4bf38720b94062dfea1bc07162652f6", "operation": "CosineSimilarity", "operators": [ "Div", "Max", "Mul", "ReduceSum", "Sqrt" ], "opset": 19, "optimization": "disabled", "output_hash": "ac4a600214113fe129ae23e5b4a7d946a3adedc0f7d91822ddb71fbc54824c5a", "outputs": {}, "parameters": { "epsilon": 1e-06 }, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.00575, "p95_ms": 0.0067874, "repeats": 7, "samples_ms": [ 0.006875, 0.006583, 0.006333, 0.00575, 0.005542, 0.005208, 0.005417 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.062083, "p95_ms": 0.0634545, "repeats": 7, "samples_ms": [ 0.063792, 0.062667, 0.06225, 0.061917, 0.062083, 0.061875, 0.061834 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-13, "rtol": 1e-12 }, "variant": "ordinary", "warnings": [] }, { "artifacts": { "bundle": "observations/48d360672bc021fcb8e33487cf9f850655b64aaf21e3ca25ae9ad34439cab7d0/reproducer.zip", "bundle_sha256": "2cec9191f4d485fad067e60903a5fd856afb1ba22e32bb625158c01c1cd56c82", "case": "artifacts/2043fa8a5ca4a2a204046677671f6981a46825220dfc305aebc71ac4a1feadd8/case.json", "inputs": "artifacts/2043fa8a5ca4a2a204046677671f6981a46825220dfc305aebc71ac4a1feadd8/inputs.npz", "manifest": "artifacts/2043fa8a5ca4a2a204046677671f6981a46825220dfc305aebc71ac4a1feadd8/manifest.json", "model": "artifacts/2043fa8a5ca4a2a204046677671f6981a46825220dfc305aebc71ac4a1feadd8/model.onnx", "outputs": "observations/48d360672bc021fcb8e33487cf9f850655b64aaf21e3ca25ae9ad34439cab7d0/outputs.npz" }, "candidate_group": "08e8a01ab19a45826736475c408df61e3ab43f9606b2cf3f2a5e96996076aebd", "case_id": "2043fa8a5ca4a2a204046677671f6981a46825220dfc305aebc71ac4a1feadd8", "category": "metrics", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 5.960464477539063e-08, "max_budget_ratio": 0.011407470606484562, "max_rel": 1.8481753519203096e-07, "max_ulp": 2, "mismatched": 0, "name": "cosine_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "reference", "name": "cosine_range", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 1 ], "backend": "reference", "elements": 8, "expected_dtype": "float32", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 2.2351741790771484e-08, "max_budget_ratio": 0.010179956589814713, "max_rel": 1.8694033568626827e-07, "max_ulp": 3, "mismatched": 0, "name": "cosine_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "cosine_range", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 1 ], "backend": "ort", "elements": 8, "expected_dtype": "float32", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } } ], "comparison": { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 5.960464477539063e-08, "max_budget_ratio": 0.009906079862459232, "max_rel": 1.1892693652298548e-07, "max_ulp": 1, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, "comparison_key": "cee65da8c7a162594a6a6a198f366294d305b0d29533f4a61c829aadebe9c4d6", "contract": "cosine", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "target": { "dtype": "float32", "shape": [ 4, 12 ] }, "x": { "dtype": "float32", "shape": [ 4, 12 ] } }, "ir_version": 10, "name": "CosineSimilarity/random-006/float32/4x12/seed-20260906", "observation_id": "48d360672bc021fcb8e33487cf9f850655b64aaf21e3ca25ae9ad34439cab7d0", "operation": "CosineSimilarity", "operators": [ "Div", "Max", "Mul", "ReduceSum", "Sqrt" ], "opset": 19, "optimization": "all", "output_hash": "f9b0cde5a0af5f48959e55355d25c305ff94c25e742115abd0ff41cffe38cb57", "outputs": {}, "parameters": { "epsilon": 1e-06 }, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.006209, "p95_ms": 0.0066210999999999996, "repeats": 7, "samples_ms": [ 0.006209, 0.006709, 0.006416, 0.006209, 0.005666, 0.005834, 0.005417 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.060125, "p95_ms": 0.0639713, "repeats": 7, "samples_ms": [ 0.061375, 0.060583, 0.060125, 0.059666, 0.059667, 0.060083, 0.065084 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-06, "rtol": 1e-05 }, "variant": "random-006", "warnings": [] }, { "artifacts": { "bundle": "observations/5fdf33673428673dd73318da05603e4b4bcf0678778d1d736d9123484ec06c6e/reproducer.zip", "bundle_sha256": "326617a848fc67ea37c877b70745d23c5b3b28950abdbd205596c3a9d262ab9c", "case": "artifacts/2043fa8a5ca4a2a204046677671f6981a46825220dfc305aebc71ac4a1feadd8/case.json", "inputs": "artifacts/2043fa8a5ca4a2a204046677671f6981a46825220dfc305aebc71ac4a1feadd8/inputs.npz", "manifest": "artifacts/2043fa8a5ca4a2a204046677671f6981a46825220dfc305aebc71ac4a1feadd8/manifest.json", "model": "artifacts/2043fa8a5ca4a2a204046677671f6981a46825220dfc305aebc71ac4a1feadd8/model.onnx", "outputs": "observations/5fdf33673428673dd73318da05603e4b4bcf0678778d1d736d9123484ec06c6e/outputs.npz" }, "candidate_group": "08e8a01ab19a45826736475c408df61e3ab43f9606b2cf3f2a5e96996076aebd", "case_id": "2043fa8a5ca4a2a204046677671f6981a46825220dfc305aebc71ac4a1feadd8", "category": "metrics", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 5.960464477539063e-08, "max_budget_ratio": 0.011407470606484562, "max_rel": 1.8481753519203096e-07, "max_ulp": 2, "mismatched": 0, "name": "cosine_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "reference", "name": "cosine_range", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 1 ], "backend": "reference", "elements": 8, "expected_dtype": "float32", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 2.2351741790771484e-08, "max_budget_ratio": 0.010179956589814713, "max_rel": 1.8694033568626827e-07, "max_ulp": 3, "mismatched": 0, "name": "cosine_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "cosine_range", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 1 ], "backend": "ort", "elements": 8, "expected_dtype": "float32", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } } ], "comparison": { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 5.960464477539063e-08, "max_budget_ratio": 0.009906079862459232, "max_rel": 1.1892693652298548e-07, "max_ulp": 1, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, "comparison_key": "8a1d89d8abaef51f06eeab71b3a8a3ac7757b20650f909694959bbcee2c199d9", "contract": "cosine", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "target": { "dtype": "float32", "shape": [ 4, 12 ] }, "x": { "dtype": "float32", "shape": [ 4, 12 ] } }, "ir_version": 10, "name": "CosineSimilarity/random-006/float32/4x12/seed-20260906", "observation_id": "5fdf33673428673dd73318da05603e4b4bcf0678778d1d736d9123484ec06c6e", "operation": "CosineSimilarity", "operators": [ "Div", "Max", "Mul", "ReduceSum", "Sqrt" ], "opset": 19, "optimization": "disabled", "output_hash": "f9b0cde5a0af5f48959e55355d25c305ff94c25e742115abd0ff41cffe38cb57", "outputs": {}, "parameters": { "epsilon": 1e-06 }, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.006084, "p95_ms": 0.009516999999999996, "repeats": 7, "samples_ms": [ 0.006208, 0.006542, 0.005875, 0.005583, 0.005292, 0.010792, 0.006084 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.061167, "p95_ms": 0.0641955, "repeats": 7, "samples_ms": [ 0.064458, 0.063583, 0.061959, 0.061167, 0.061041, 0.061, 0.061125 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-06, "rtol": 1e-05 }, "variant": "random-006", "warnings": [] }, { "artifacts": { "bundle": "observations/5a55e2b9bcf5254fdd41980df80261abf9d8e265ec062a32b46ee57f03f9413f/reproducer.zip", "bundle_sha256": "4994adb39101c6cf2f666d920188d9827bc62ece2264daa96a685a6e12c15ed2", "case": "artifacts/e20c76044fe9006c1a3e878647860be6a6acc851a467de4e0960fc020aea89f5/case.json", "inputs": "artifacts/e20c76044fe9006c1a3e878647860be6a6acc851a467de4e0960fc020aea89f5/inputs.npz", "manifest": "artifacts/e20c76044fe9006c1a3e878647860be6a6acc851a467de4e0960fc020aea89f5/manifest.json", "model": "artifacts/e20c76044fe9006c1a3e878647860be6a6acc851a467de4e0960fc020aea89f5/model.onnx", "outputs": "observations/5a55e2b9bcf5254fdd41980df80261abf9d8e265ec062a32b46ee57f03f9413f/outputs.npz" }, "candidate_group": "65b2f379f01a7b4c0d17a483d296a245bd1abc89d3cebcb426c0d78b0ea48ad2", "case_id": "e20c76044fe9006c1a3e878647860be6a6acc851a467de4e0960fc020aea89f5", "category": "metrics", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 2.9802322387695312e-08, "max_budget_ratio": 0.022359133041609254, "max_rel": 2.9182674111690624e-06, "max_ulp": 26, "mismatched": 0, "name": "cosine_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "reference", "name": "cosine_range", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 1 ], "backend": "reference", "elements": 8, "expected_dtype": "float32", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 2.1420419216156006e-08, "max_budget_ratio": 0.019779233075269725, "max_rel": 2.581544248341863e-06, "max_ulp": 23, "mismatched": 0, "name": "cosine_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "cosine_range", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 1 ], "backend": "ort", "elements": 8, "expected_dtype": "float32", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } } ], "comparison": { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 1.4901161193847656e-08, "max_budget_ratio": 0.0077317282657194605, "max_rel": 4.992020670959194e-07, "max_ulp": 6, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, "comparison_key": "e41f2c0da485f865c79cf011b6758ec3b16e1a05ddcd237516b76c7417509c60", "contract": "cosine", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "target": { "dtype": "float32", "shape": [ 4, 92 ] }, "x": { "dtype": "float32", "shape": [ 4, 92 ] } }, "ir_version": 10, "name": "CosineSimilarity/random-013/float32/4x92/seed-20260906", "observation_id": "5a55e2b9bcf5254fdd41980df80261abf9d8e265ec062a32b46ee57f03f9413f", "operation": "CosineSimilarity", "operators": [ "Div", "Max", "Mul", "ReduceSum", "Sqrt" ], "opset": 19, "optimization": "all", "output_hash": "46430ffa7a079499bc26c4cc47e7a0d30211d2ae4364770978b8f7a65c3d57e6", "outputs": {}, "parameters": { "epsilon": 1e-06 }, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.005542, "p95_ms": 0.006966899999999999, "repeats": 7, "samples_ms": [ 0.007167, 0.0065, 0.00625, 0.005542, 0.005417, 0.005333, 0.0055 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.061292, "p95_ms": 0.0619285, "repeats": 7, "samples_ms": [ 0.062041, 0.061666, 0.061, 0.060584, 0.060917, 0.061375, 0.061292 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-06, "rtol": 1e-05 }, "variant": "random-013", "warnings": [] }, { "artifacts": { "bundle": "observations/c3ebc10623daa2e63b73ce532f30dd4b7b636b67c2afea646a8c59e273f95f2a/reproducer.zip", "bundle_sha256": "251c6a7729cab5bb4bed23d05400220ff38544380e83783a4122cc8664e138f4", "case": "artifacts/e20c76044fe9006c1a3e878647860be6a6acc851a467de4e0960fc020aea89f5/case.json", "inputs": "artifacts/e20c76044fe9006c1a3e878647860be6a6acc851a467de4e0960fc020aea89f5/inputs.npz", "manifest": "artifacts/e20c76044fe9006c1a3e878647860be6a6acc851a467de4e0960fc020aea89f5/manifest.json", "model": "artifacts/e20c76044fe9006c1a3e878647860be6a6acc851a467de4e0960fc020aea89f5/model.onnx", "outputs": "observations/c3ebc10623daa2e63b73ce532f30dd4b7b636b67c2afea646a8c59e273f95f2a/outputs.npz" }, "candidate_group": "65b2f379f01a7b4c0d17a483d296a245bd1abc89d3cebcb426c0d78b0ea48ad2", "case_id": "e20c76044fe9006c1a3e878647860be6a6acc851a467de4e0960fc020aea89f5", "category": "metrics", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 2.9802322387695312e-08, "max_budget_ratio": 0.022359133041609254, "max_rel": 2.9182674111690624e-06, "max_ulp": 26, "mismatched": 0, "name": "cosine_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "reference", "name": "cosine_range", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 1 ], "backend": "reference", "elements": 8, "expected_dtype": "float32", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 2.1420419216156006e-08, "max_budget_ratio": 0.019779233075269725, "max_rel": 2.581544248341863e-06, "max_ulp": 23, "mismatched": 0, "name": "cosine_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "cosine_range", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 1 ], "backend": "ort", "elements": 8, "expected_dtype": "float32", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } } ], "comparison": { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 1.4901161193847656e-08, "max_budget_ratio": 0.0077317282657194605, "max_rel": 4.992020670959194e-07, "max_ulp": 6, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, "comparison_key": "7cb71c4f0719737dbd47ca0cfa24e8bdc29c768d76abdd042d2b20ffd6c2948e", "contract": "cosine", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "target": { "dtype": "float32", "shape": [ 4, 92 ] }, "x": { "dtype": "float32", "shape": [ 4, 92 ] } }, "ir_version": 10, "name": "CosineSimilarity/random-013/float32/4x92/seed-20260906", "observation_id": "c3ebc10623daa2e63b73ce532f30dd4b7b636b67c2afea646a8c59e273f95f2a", "operation": "CosineSimilarity", "operators": [ "Div", "Max", "Mul", "ReduceSum", "Sqrt" ], "opset": 19, "optimization": "disabled", "output_hash": "46430ffa7a079499bc26c4cc47e7a0d30211d2ae4364770978b8f7a65c3d57e6", "outputs": {}, "parameters": { "epsilon": 1e-06 }, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.0065, "p95_ms": 0.007091999999999999, "repeats": 7, "samples_ms": [ 0.007167, 0.006917, 0.006709, 0.006375, 0.006084, 0.005959, 0.0065 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.0625, "p95_ms": 0.0649999, "repeats": 7, "samples_ms": [ 0.065125, 0.064708, 0.062792, 0.061917, 0.061542, 0.0625, 0.061334 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-06, "rtol": 1e-05 }, "variant": "random-013", "warnings": [] }, { "artifacts": { "bundle": "observations/20ba41a8d2d21de61d49a3c22b72d82b8c06169b73eb3a5d1fd73d3b4b7864af/reproducer.zip", "bundle_sha256": "3316bd53f2ca9450c1e37c0ecef907c0e45145dc7abe1180b87251fae25073ef", "case": "artifacts/ed18f6c0ea2aa72046324a51295a5fef1b5c4b44fd73dbac47ea59db3f1ca71e/case.json", "inputs": "artifacts/ed18f6c0ea2aa72046324a51295a5fef1b5c4b44fd73dbac47ea59db3f1ca71e/inputs.npz", "manifest": "artifacts/ed18f6c0ea2aa72046324a51295a5fef1b5c4b44fd73dbac47ea59db3f1ca71e/manifest.json", "model": "artifacts/ed18f6c0ea2aa72046324a51295a5fef1b5c4b44fd73dbac47ea59db3f1ca71e/model.onnx", "outputs": "observations/20ba41a8d2d21de61d49a3c22b72d82b8c06169b73eb3a5d1fd73d3b4b7864af/outputs.npz" }, "candidate_group": "a520c5e2684f214ae3db6c95c2470e9b52c18fd6526f18b41681578bd0590eac", "case_id": "ed18f6c0ea2aa72046324a51295a5fef1b5c4b44fd73dbac47ea59db3f1ca71e", "category": "metrics", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 1.4901161193847656e-08, "max_budget_ratio": 0.0060204890245762616, "max_rel": 1.9668264325737925e-06, "max_ulp": 22, "mismatched": 0, "name": "cosine_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "reference", "name": "cosine_range", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 1 ], "backend": "reference", "elements": 8, "expected_dtype": "float32", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 6.51925802230835e-09, "max_budget_ratio": 0.005733494399531681, "max_rel": 5.364072088837616e-07, "max_ulp": 7, "mismatched": 0, "name": "cosine_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "cosine_range", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 3.725290298461914e-09, "max_budget_ratio": 0.003630733790841554, "max_rel": 1.4304184564032592e-06, "max_ulp": 16, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 1.4901161193847656e-08, "max_budget_ratio": 0.0060204890245762616, "max_rel": 6.25808074676426e-07, "max_ulp": 7, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 1 ], "backend": "ort", "elements": 8, "expected_dtype": "float32", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } } ], "comparison": { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 1.4901161193847656e-08, "max_budget_ratio": 0.006020489387039165, "max_rel": 1.4304164103092256e-06, "max_ulp": 16, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, "comparison_key": "1f9bd4e1e10ba131f41e1881d364cf6a4aed289e9dfc368e2ceb6da05b2c7ea5", "contract": "cosine", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "target": { "dtype": "float32", "shape": [ 4, 55 ] }, "x": { "dtype": "float32", "shape": [ 4, 55 ] } }, "ir_version": 10, "name": "CosineSimilarity/random-020/float32/4x55/seed-20260906", "observation_id": "20ba41a8d2d21de61d49a3c22b72d82b8c06169b73eb3a5d1fd73d3b4b7864af", "operation": "CosineSimilarity", "operators": [ "Div", "Max", "Mul", "ReduceSum", "Sqrt" ], "opset": 19, "optimization": "all", "output_hash": "d06f1d08cb2746854f7af117ab146128c16a6b7da70653e30040c71d99ccbed6", "outputs": {}, "parameters": { "epsilon": 1e-06 }, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.005958, "p95_ms": 0.006796399999999999, "repeats": 7, "samples_ms": [ 0.006417, 0.006959, 0.006209, 0.005958, 0.005708, 0.005459, 0.005708 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.062834, "p95_ms": 0.0633213, "repeats": 7, "samples_ms": [ 0.063459, 0.063, 0.062834, 0.06225, 0.062291, 0.062875, 0.061333 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-06, "rtol": 1e-05 }, "variant": "random-020", "warnings": [] }, { "artifacts": { "bundle": "observations/1c657b5ac4484a89fb459678155007e73af787e0cbf91a562b1017af6eeb24c0/reproducer.zip", "bundle_sha256": "cc17e20dcfee5133df8e6df2a2c8a21bedba26b5b28e6b7f73edb230f99c3827", "case": "artifacts/ed18f6c0ea2aa72046324a51295a5fef1b5c4b44fd73dbac47ea59db3f1ca71e/case.json", "inputs": "artifacts/ed18f6c0ea2aa72046324a51295a5fef1b5c4b44fd73dbac47ea59db3f1ca71e/inputs.npz", "manifest": "artifacts/ed18f6c0ea2aa72046324a51295a5fef1b5c4b44fd73dbac47ea59db3f1ca71e/manifest.json", "model": "artifacts/ed18f6c0ea2aa72046324a51295a5fef1b5c4b44fd73dbac47ea59db3f1ca71e/model.onnx", "outputs": "observations/1c657b5ac4484a89fb459678155007e73af787e0cbf91a562b1017af6eeb24c0/outputs.npz" }, "candidate_group": "a520c5e2684f214ae3db6c95c2470e9b52c18fd6526f18b41681578bd0590eac", "case_id": "ed18f6c0ea2aa72046324a51295a5fef1b5c4b44fd73dbac47ea59db3f1ca71e", "category": "metrics", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 1.4901161193847656e-08, "max_budget_ratio": 0.0060204890245762616, "max_rel": 1.9668264325737925e-06, "max_ulp": 22, "mismatched": 0, "name": "cosine_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "reference", "name": "cosine_range", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 1 ], "backend": "reference", "elements": 8, "expected_dtype": "float32", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 6.51925802230835e-09, "max_budget_ratio": 0.005733494399531681, "max_rel": 5.364072088837616e-07, "max_ulp": 7, "mismatched": 0, "name": "cosine_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "cosine_range", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 3.725290298461914e-09, "max_budget_ratio": 0.003630733790841554, "max_rel": 1.4304184564032592e-06, "max_ulp": 16, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 1.4901161193847656e-08, "max_budget_ratio": 0.0060204890245762616, "max_rel": 6.25808074676426e-07, "max_ulp": 7, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 1 ], "backend": "ort", "elements": 8, "expected_dtype": "float32", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } } ], "comparison": { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 1.4901161193847656e-08, "max_budget_ratio": 0.006020489387039165, "max_rel": 1.4304164103092256e-06, "max_ulp": 16, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, "comparison_key": "d8692814e1f59eb28f655ec07c6fa48e27735fcdd184d02b3c37a09c92204616", "contract": "cosine", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "target": { "dtype": "float32", "shape": [ 4, 55 ] }, "x": { "dtype": "float32", "shape": [ 4, 55 ] } }, "ir_version": 10, "name": "CosineSimilarity/random-020/float32/4x55/seed-20260906", "observation_id": "1c657b5ac4484a89fb459678155007e73af787e0cbf91a562b1017af6eeb24c0", "operation": "CosineSimilarity", "operators": [ "Div", "Max", "Mul", "ReduceSum", "Sqrt" ], "opset": 19, "optimization": "disabled", "output_hash": "d06f1d08cb2746854f7af117ab146128c16a6b7da70653e30040c71d99ccbed6", "outputs": {}, "parameters": { "epsilon": 1e-06 }, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.005333, "p95_ms": 0.0059874, "repeats": 7, "samples_ms": [ 0.005958, 0.006, 0.005458, 0.005292, 0.00525, 0.005333, 0.005 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.061709, "p95_ms": 0.0636956, "repeats": 7, "samples_ms": [ 0.063833, 0.063375, 0.060292, 0.057875, 0.061292, 0.06275, 0.061709 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-06, "rtol": 1e-05 }, "variant": "random-020", "warnings": [] }, { "artifacts": { "bundle": "observations/d36ef9b045fd03462eb8468d6edfed45e49527b6c07d4989b4d79fb5a4ecf13b/reproducer.zip", "bundle_sha256": "7962053905e5929c2ab654995f8b850b9b7c326b734d9719e2258fd690ab6275", "case": "artifacts/2965f56cd92655f0aa10d8740fb3e5271e12daa553264b4ed7ea7148a9072ab3/case.json", "inputs": "artifacts/2965f56cd92655f0aa10d8740fb3e5271e12daa553264b4ed7ea7148a9072ab3/inputs.npz", "manifest": "artifacts/2965f56cd92655f0aa10d8740fb3e5271e12daa553264b4ed7ea7148a9072ab3/manifest.json", "model": "artifacts/2965f56cd92655f0aa10d8740fb3e5271e12daa553264b4ed7ea7148a9072ab3/model.onnx", "outputs": "observations/d36ef9b045fd03462eb8468d6edfed45e49527b6c07d4989b4d79fb5a4ecf13b/outputs.npz" }, "candidate_group": "e569bbf2093659b4d8fce8abfe870ba83155bb8240327f238642ec2147c0dd74", "case_id": "2965f56cd92655f0aa10d8740fb3e5271e12daa553264b4ed7ea7148a9072ab3", "category": "metrics", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": [ 0, 0 ], "max_abs": 1.9234619140625, "max_budget_ratio": 1636.0710206624442, "max_rel": 14.233965672990063, "max_ulp": 4043, "mismatched": 4, "name": "cosine_float64_oracle", "nonfinite": 0, "numerical_divergence": true, "passed": false, "reason": "tolerance_exceeded", "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "backend": "reference", "name": "cosine_range", "passed": false }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 8, 1 ], "backend": "reference", "elements": 8, "expected_dtype": "float16", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "cosine_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "backend": "ort", "name": "cosine_range", "passed": true }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 8, 1 ], "backend": "ort", "elements": 8, "expected_dtype": "float16", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } } ], "comparison": { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": [ 0, 0 ], "max_abs": 1.9234619140625, "max_budget_ratio": 178.21435034382915, "max_rel": 0.9343572106261859, "max_ulp": 4043, "mismatched": 4, "nonfinite": 0, "numerical_divergence": true, "passed": false, "reason": "tolerance_exceeded", "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, "comparison_key": "947860fb95413f3bf5409020ee901dd9df512e85625d3535f93532379871b2b1", "contract": "cosine", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "target": { "dtype": "float16", "shape": [ 4, 17 ] }, "x": { "dtype": "float16", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "CosineSimilarity/tiny/float16/4x17/seed-20260906", "observation_id": "d36ef9b045fd03462eb8468d6edfed45e49527b6c07d4989b4d79fb5a4ecf13b", "operation": "CosineSimilarity", "operators": [ "Div", "Max", "Mul", "ReduceSum", "Sqrt" ], "opset": 19, "optimization": "all", "output_hash": "f2f7df658a272da02a65adab7c9631116102994576c301998abd85847fa141f8", "outputs": {}, "parameters": { "epsilon": 1e-06 }, "regression": "not_assessed", "seed": 20260906, "status": "divergence", "timings": { "ort": { "median_ms": 0.006958, "p95_ms": 0.007717, "repeats": 7, "samples_ms": [ 0.007792, 0.007542, 0.007292, 0.006958, 0.006833, 0.006708, 0.006375 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.066833, "p95_ms": 0.07152869999999999, "repeats": 7, "samples_ms": [ 0.070625, 0.071916, 0.06775, 0.066375, 0.066833, 0.066375, 0.065667 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 0.0005, "rtol": 0.005 }, "variant": "tiny", "warnings": [ { "backend": "reference", "message": "underflow encountered in multiply" } ] }, { "artifacts": { "bundle": "observations/f4014f1cd63164522100d26f263c3d7355878f1424e703c40b8da199c80f2423/reproducer.zip", "bundle_sha256": "bd0092cf93659185bc4c8c5434f6e63d5d188ae5078dd410b55218d2dba32974", "case": "artifacts/2965f56cd92655f0aa10d8740fb3e5271e12daa553264b4ed7ea7148a9072ab3/case.json", "inputs": "artifacts/2965f56cd92655f0aa10d8740fb3e5271e12daa553264b4ed7ea7148a9072ab3/inputs.npz", "manifest": "artifacts/2965f56cd92655f0aa10d8740fb3e5271e12daa553264b4ed7ea7148a9072ab3/manifest.json", "model": "artifacts/2965f56cd92655f0aa10d8740fb3e5271e12daa553264b4ed7ea7148a9072ab3/model.onnx", "outputs": "observations/f4014f1cd63164522100d26f263c3d7355878f1424e703c40b8da199c80f2423/outputs.npz" }, "candidate_group": "e569bbf2093659b4d8fce8abfe870ba83155bb8240327f238642ec2147c0dd74", "case_id": "2965f56cd92655f0aa10d8740fb3e5271e12daa553264b4ed7ea7148a9072ab3", "category": "metrics", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": [ 0, 0 ], "max_abs": 1.9234619140625, "max_budget_ratio": 1636.0710206624442, "max_rel": 14.233965672990063, "max_ulp": 4043, "mismatched": 4, "name": "cosine_float64_oracle", "nonfinite": 0, "numerical_divergence": true, "passed": false, "reason": "tolerance_exceeded", "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "backend": "reference", "name": "cosine_range", "passed": false }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 8, 1 ], "backend": "reference", "elements": 8, "expected_dtype": "float16", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "cosine_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "backend": "ort", "name": "cosine_range", "passed": true }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 8, 1 ], "backend": "ort", "elements": 8, "expected_dtype": "float16", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } } ], "comparison": { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": [ 0, 0 ], "max_abs": 1.9234619140625, "max_budget_ratio": 178.21435034382915, "max_rel": 0.9343572106261859, "max_ulp": 4043, "mismatched": 4, "nonfinite": 0, "numerical_divergence": true, "passed": false, "reason": "tolerance_exceeded", "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, "comparison_key": "ddf661a85f2ea8fda193f4de6c636c48b305792c2f5f1f5981cc0a7a5d1424be", "contract": "cosine", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "target": { "dtype": "float16", "shape": [ 4, 17 ] }, "x": { "dtype": "float16", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "CosineSimilarity/tiny/float16/4x17/seed-20260906", "observation_id": "f4014f1cd63164522100d26f263c3d7355878f1424e703c40b8da199c80f2423", "operation": "CosineSimilarity", "operators": [ "Div", "Max", "Mul", "ReduceSum", "Sqrt" ], "opset": 19, "optimization": "disabled", "output_hash": "f2f7df658a272da02a65adab7c9631116102994576c301998abd85847fa141f8", "outputs": {}, "parameters": { "epsilon": 1e-06 }, "regression": "not_assessed", "seed": 20260906, "status": "divergence", "timings": { "ort": { "median_ms": 0.0075, "p95_ms": 0.0079877, "repeats": 7, "samples_ms": [ 0.007959, 0.008, 0.007542, 0.0075, 0.006834, 0.007, 0.007167 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.068125, "p95_ms": 0.07130869999999999, "repeats": 7, "samples_ms": [ 0.072584, 0.068333, 0.067209, 0.068125, 0.067542, 0.06825, 0.066583 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 0.0005, "rtol": 0.005 }, "variant": "tiny", "warnings": [ { "backend": "reference", "message": "underflow encountered in multiply" } ] }, { "artifacts": { "bundle": "observations/2218e5d8ffcab9f8a915138cb57d7ec9f1f95b0a716daa612fcbd89826f52460/reproducer.zip", "bundle_sha256": "47e3c672d2be3fe5f94e8af299d84c793aece5c9ba059fae98d03cd92c35760d", "case": "artifacts/1bcf89d04223d5b13f25256e69264f8c4ecbd9612db40bb2ba9d94f02fe7db97/case.json", "inputs": "artifacts/1bcf89d04223d5b13f25256e69264f8c4ecbd9612db40bb2ba9d94f02fe7db97/inputs.npz", "manifest": "artifacts/1bcf89d04223d5b13f25256e69264f8c4ecbd9612db40bb2ba9d94f02fe7db97/manifest.json", "model": "artifacts/1bcf89d04223d5b13f25256e69264f8c4ecbd9612db40bb2ba9d94f02fe7db97/model.onnx", "outputs": "observations/2218e5d8ffcab9f8a915138cb57d7ec9f1f95b0a716daa612fcbd89826f52460/outputs.npz" }, "candidate_group": "ee6464cc6aeef066278bff54b6e73ad1ae019ad77e70b6bb7f470da1df185bcd", "case_id": "1bcf89d04223d5b13f25256e69264f8c4ecbd9612db40bb2ba9d94f02fe7db97", "category": "metrics", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 2.9802322387695312e-08, "max_budget_ratio": 0.006397068852816329, "max_rel": 1.1209162638560663e-07, "max_ulp": 1, "mismatched": 0, "name": "cosine_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "reference", "name": "cosine_range", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 1 ], "backend": "reference", "elements": 8, "expected_dtype": "float32", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 2.9802322387695312e-08, "max_budget_ratio": 0.006397068852816329, "max_rel": 1.1209162638560663e-07, "max_ulp": 1, "mismatched": 0, "name": "cosine_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "cosine_range", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 2.9802322387695312e-08, "max_budget_ratio": 0.012794136887182912, "max_rel": 2.2418322764215065e-07, "max_ulp": 2, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 5.960464477539063e-08, "max_budget_ratio": 0.0179617375832888, "max_rel": 2.5709131767912867e-07, "max_ulp": 4, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 1 ], "backend": "ort", "elements": 8, "expected_dtype": "float32", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } } ], "comparison": { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 2.9802322387695312e-08, "max_budget_ratio": 0.012794138524082508, "max_rel": 2.2418327790028148e-07, "max_ulp": 2, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, "comparison_key": "e11c39b0f7ca9e36fd80d4e788623a57b15bc35e89a7f521bfc6dd9a15e9e164", "contract": "cosine", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "target": { "dtype": "float32", "shape": [ 4, 17 ] }, "x": { "dtype": "float32", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "CosineSimilarity/tiny/float32/4x17/seed-20260906", "observation_id": "2218e5d8ffcab9f8a915138cb57d7ec9f1f95b0a716daa612fcbd89826f52460", "operation": "CosineSimilarity", "operators": [ "Div", "Max", "Mul", "ReduceSum", "Sqrt" ], "opset": 19, "optimization": "all", "output_hash": "0697442f5dbe8deac205f63a8d325c3131176c507fc97fe5da045cd1d90b61ab", "outputs": {}, "parameters": { "epsilon": 1e-06 }, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.005625, "p95_ms": 0.0064624999999999995, "repeats": 7, "samples_ms": [ 0.006375, 0.0065, 0.005958, 0.005541, 0.005375, 0.005625, 0.005625 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.06125, "p95_ms": 0.0640625, "repeats": 7, "samples_ms": [ 0.06425, 0.06175, 0.063625, 0.06125, 0.060208, 0.061, 0.059584 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-06, "rtol": 1e-05 }, "variant": "tiny", "warnings": [] }, { "artifacts": { "bundle": "observations/4ff7f0288bb451614fab63421adf1105434f67cfd1d3f4505a854f9a8f850f40/reproducer.zip", "bundle_sha256": "7e62c0b0581c97016db732312c01b2fb3fc33c7a80e694bd23657471c9d7469b", "case": "artifacts/1bcf89d04223d5b13f25256e69264f8c4ecbd9612db40bb2ba9d94f02fe7db97/case.json", "inputs": "artifacts/1bcf89d04223d5b13f25256e69264f8c4ecbd9612db40bb2ba9d94f02fe7db97/inputs.npz", "manifest": "artifacts/1bcf89d04223d5b13f25256e69264f8c4ecbd9612db40bb2ba9d94f02fe7db97/manifest.json", "model": "artifacts/1bcf89d04223d5b13f25256e69264f8c4ecbd9612db40bb2ba9d94f02fe7db97/model.onnx", "outputs": "observations/4ff7f0288bb451614fab63421adf1105434f67cfd1d3f4505a854f9a8f850f40/outputs.npz" }, "candidate_group": "ee6464cc6aeef066278bff54b6e73ad1ae019ad77e70b6bb7f470da1df185bcd", "case_id": "1bcf89d04223d5b13f25256e69264f8c4ecbd9612db40bb2ba9d94f02fe7db97", "category": "metrics", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 2.9802322387695312e-08, "max_budget_ratio": 0.006397068852816329, "max_rel": 1.1209162638560663e-07, "max_ulp": 1, "mismatched": 0, "name": "cosine_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "reference", "name": "cosine_range", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 1 ], "backend": "reference", "elements": 8, "expected_dtype": "float32", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 2.9802322387695312e-08, "max_budget_ratio": 0.006397068852816329, "max_rel": 1.1209162638560663e-07, "max_ulp": 1, "mismatched": 0, "name": "cosine_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "cosine_range", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 2.9802322387695312e-08, "max_budget_ratio": 0.012794136887182912, "max_rel": 2.2418322764215065e-07, "max_ulp": 2, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 5.960464477539063e-08, "max_budget_ratio": 0.0179617375832888, "max_rel": 2.5709131767912867e-07, "max_ulp": 4, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 1 ], "backend": "ort", "elements": 8, "expected_dtype": "float32", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } } ], "comparison": { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 2.9802322387695312e-08, "max_budget_ratio": 0.012794138524082508, "max_rel": 2.2418327790028148e-07, "max_ulp": 2, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, "comparison_key": "a7fbeb3b0b33533de21d08237b709814557844e150a367ce386b5f66bf34ff77", "contract": "cosine", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "target": { "dtype": "float32", "shape": [ 4, 17 ] }, "x": { "dtype": "float32", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "CosineSimilarity/tiny/float32/4x17/seed-20260906", "observation_id": "4ff7f0288bb451614fab63421adf1105434f67cfd1d3f4505a854f9a8f850f40", "operation": "CosineSimilarity", "operators": [ "Div", "Max", "Mul", "ReduceSum", "Sqrt" ], "opset": 19, "optimization": "disabled", "output_hash": "0697442f5dbe8deac205f63a8d325c3131176c507fc97fe5da045cd1d90b61ab", "outputs": {}, "parameters": { "epsilon": 1e-06 }, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.00575, "p95_ms": 0.0068956, "repeats": 7, "samples_ms": [ 0.00675, 0.006958, 0.005959, 0.00575, 0.005459, 0.005666, 0.005708 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.064042, "p95_ms": 0.0652787, "repeats": 7, "samples_ms": [ 0.064042, 0.06525, 0.062, 0.06225, 0.064292, 0.065291, 0.061917 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-06, "rtol": 1e-05 }, "variant": "tiny", "warnings": [] }, { "artifacts": { "bundle": "observations/a485700642f970ab64a48daeeb49490cef7c5b0d1067e9ead3579e1de1196563/reproducer.zip", "bundle_sha256": "5a3a00dfc4fad9ae2a2543d8d20187468f025ed72aa046a3bbe91ed57984092f", "case": "artifacts/ce019763259e96bd3f1842de743cefd7f3cee0cb6edd8f61b77df37e76bd8488/case.json", "inputs": "artifacts/ce019763259e96bd3f1842de743cefd7f3cee0cb6edd8f61b77df37e76bd8488/inputs.npz", "manifest": "artifacts/ce019763259e96bd3f1842de743cefd7f3cee0cb6edd8f61b77df37e76bd8488/manifest.json", "model": "artifacts/ce019763259e96bd3f1842de743cefd7f3cee0cb6edd8f61b77df37e76bd8488/model.onnx", "outputs": "observations/a485700642f970ab64a48daeeb49490cef7c5b0d1067e9ead3579e1de1196563/outputs.npz" }, "candidate_group": "bef40c738988b27fa9be4571652bc63c6527407eef9066973f7d5eb4f591fa67", "case_id": "ce019763259e96bd3f1842de743cefd7f3cee0cb6edd8f61b77df37e76bd8488", "category": "metrics", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "cosine_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "backend": "reference", "name": "cosine_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 8, 1 ], "backend": "reference", "elements": 8, "expected_dtype": "float64", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 5.551115123125783e-17, "max_budget_ratio": 0.00013943694563291314, "max_rel": 2.6568040346172733e-16, "max_ulp": 2, "mismatched": 0, "name": "cosine_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "backend": "ort", "name": "cosine_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 5.551115123125783e-17, "max_budget_ratio": 0.0001394369456329132, "max_rel": 1.8621070415930203e-16, "max_ulp": 1, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 6.245004513516506e-17, "max_budget_ratio": 0.0004951737172932836, "max_rel": 2.3911236311555466e-15, "max_ulp": 18, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 8, 1 ], "backend": "ort", "elements": 8, "expected_dtype": "float64", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } } ], "comparison": { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 5.551115123125783e-17, "max_budget_ratio": 0.00013943694563291314, "max_rel": 2.6568040346172733e-16, "max_ulp": 2, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, "comparison_key": "62af024b8c14f823abece73bf8063b1cdd3ad6f8741c8596845e4e01595f8b81", "contract": "cosine", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "target": { "dtype": "float64", "shape": [ 4, 17 ] }, "x": { "dtype": "float64", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "CosineSimilarity/tiny/float64/4x17/seed-20260906", "observation_id": "a485700642f970ab64a48daeeb49490cef7c5b0d1067e9ead3579e1de1196563", "operation": "CosineSimilarity", "operators": [ "Div", "Max", "Mul", "ReduceSum", "Sqrt" ], "opset": 19, "optimization": "all", "output_hash": "58c2fc52eb959cef86cf85ca72c80fa83bd44e85f35198bd20ffecacd35268eb", "outputs": {}, "parameters": { "epsilon": 1e-06 }, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.006542, "p95_ms": 0.009491199999999998, "repeats": 7, "samples_ms": [ 0.006584, 0.00675, 0.006542, 0.005833, 0.006125, 0.005791, 0.010666 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.063416, "p95_ms": 0.0676837, "repeats": 7, "samples_ms": [ 0.063833, 0.063583, 0.062625, 0.062833, 0.063209, 0.063416, 0.069334 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-13, "rtol": 1e-12 }, "variant": "tiny", "warnings": [] }, { "artifacts": { "bundle": "observations/368d40cb3ec6807c1c9fae979413f27e09c6684bb71c9224f5c7cc64548c634f/reproducer.zip", "bundle_sha256": "7de1790447afde140c49dc702e34eb950a0db7578e500ce674600ebcf0a58f7c", "case": "artifacts/ce019763259e96bd3f1842de743cefd7f3cee0cb6edd8f61b77df37e76bd8488/case.json", "inputs": "artifacts/ce019763259e96bd3f1842de743cefd7f3cee0cb6edd8f61b77df37e76bd8488/inputs.npz", "manifest": "artifacts/ce019763259e96bd3f1842de743cefd7f3cee0cb6edd8f61b77df37e76bd8488/manifest.json", "model": "artifacts/ce019763259e96bd3f1842de743cefd7f3cee0cb6edd8f61b77df37e76bd8488/model.onnx", "outputs": "observations/368d40cb3ec6807c1c9fae979413f27e09c6684bb71c9224f5c7cc64548c634f/outputs.npz" }, "candidate_group": "bef40c738988b27fa9be4571652bc63c6527407eef9066973f7d5eb4f591fa67", "case_id": "ce019763259e96bd3f1842de743cefd7f3cee0cb6edd8f61b77df37e76bd8488", "category": "metrics", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "cosine_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "backend": "reference", "name": "cosine_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 8, 1 ], "backend": "reference", "elements": 8, "expected_dtype": "float64", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 5.551115123125783e-17, "max_budget_ratio": 0.00013943694563291314, "max_rel": 2.6568040346172733e-16, "max_ulp": 2, "mismatched": 0, "name": "cosine_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "backend": "ort", "name": "cosine_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 5.551115123125783e-17, "max_budget_ratio": 0.0001394369456329132, "max_rel": 1.8621070415930203e-16, "max_ulp": 1, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 6.245004513516506e-17, "max_budget_ratio": 0.0004951737172932836, "max_rel": 2.3911236311555466e-15, "max_ulp": 18, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 8, 1 ], "backend": "ort", "elements": 8, "expected_dtype": "float64", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } } ], "comparison": { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 5.551115123125783e-17, "max_budget_ratio": 0.00013943694563291314, "max_rel": 2.6568040346172733e-16, "max_ulp": 2, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, "comparison_key": "025d5ade3991fa7038308266128f3e4511261f12fa4851828780e0465a8f5746", "contract": "cosine", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "target": { "dtype": "float64", "shape": [ 4, 17 ] }, "x": { "dtype": "float64", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "CosineSimilarity/tiny/float64/4x17/seed-20260906", "observation_id": "368d40cb3ec6807c1c9fae979413f27e09c6684bb71c9224f5c7cc64548c634f", "operation": "CosineSimilarity", "operators": [ "Div", "Max", "Mul", "ReduceSum", "Sqrt" ], "opset": 19, "optimization": "disabled", "output_hash": "58c2fc52eb959cef86cf85ca72c80fa83bd44e85f35198bd20ffecacd35268eb", "outputs": {}, "parameters": { "epsilon": 1e-06 }, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.005667, "p95_ms": 0.0064587, "repeats": 7, "samples_ms": [ 0.006459, 0.006458, 0.006208, 0.005667, 0.005209, 0.005125, 0.005167 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.062291, "p95_ms": 0.0655169, "repeats": 7, "samples_ms": [ 0.066167, 0.062291, 0.064, 0.061542, 0.060792, 0.060959, 0.063458 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-13, "rtol": 1e-12 }, "variant": "tiny", "warnings": [] }, { "artifacts": { "bundle": "observations/ec2702489b4f82975c92aebdf03d8d9ddd6afc7056132256f1455c9aa27d6d5b/reproducer.zip", "bundle_sha256": "bd6b9a08ffa84c1fcbf090205f3d17ad5f99f00334b31e87676fe965539886f6", "case": "artifacts/10dbe9dccca062e40c515ac3b06b1c9e086a6eb7eef5761c708b7c75d41eb5be/case.json", "inputs": "artifacts/10dbe9dccca062e40c515ac3b06b1c9e086a6eb7eef5761c708b7c75d41eb5be/inputs.npz", "manifest": "artifacts/10dbe9dccca062e40c515ac3b06b1c9e086a6eb7eef5761c708b7c75d41eb5be/manifest.json", "model": "artifacts/10dbe9dccca062e40c515ac3b06b1c9e086a6eb7eef5761c708b7c75d41eb5be/model.onnx", "outputs": "observations/ec2702489b4f82975c92aebdf03d8d9ddd6afc7056132256f1455c9aa27d6d5b/outputs.npz" }, "candidate_group": "9fca66aae60d73e009d8c9e7aa8b65196b60ae7806561dd919865ff15f17ce7f", "case_id": "10dbe9dccca062e40c515ac3b06b1c9e086a6eb7eef5761c708b7c75d41eb5be", "category": "metrics", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0009765625, "max_budget_ratio": 0.1775568181818182, "max_rel": 0.0009765625, "max_ulp": 1, "mismatched": 0, "name": "cosine_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "backend": "reference", "name": "cosine_range", "passed": true }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 8, 1 ], "backend": "reference", "elements": 8, "expected_dtype": "float16", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "cosine_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "backend": "ort", "name": "cosine_range", "passed": true }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 8, 1 ], "backend": "ort", "elements": 8, "expected_dtype": "float16", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } } ], "comparison": { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0009765625, "max_budget_ratio": 0.17739932588256163, "max_rel": 0.000975609756097561, "max_ulp": 1, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, "comparison_key": "dd7151756e20a981d853c51a2d080c7739d96415bd04a7436227f90008d05e90", "contract": "cosine", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "target": { "dtype": "float16", "shape": [ 4, 257 ] }, "x": { "dtype": "float16", "shape": [ 4, 257 ] } }, "ir_version": 10, "name": "CosineSimilarity/wide/float16/4x257/seed-20260906", "observation_id": "ec2702489b4f82975c92aebdf03d8d9ddd6afc7056132256f1455c9aa27d6d5b", "operation": "CosineSimilarity", "operators": [ "Div", "Max", "Mul", "ReduceSum", "Sqrt" ], "opset": 19, "optimization": "all", "output_hash": "43ef0964bdc7e3a00a5c45a014c64d80a416ef57052e53ca39cd4cec748cf27c", "outputs": {}, "parameters": { "epsilon": 1e-06 }, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.008333, "p95_ms": 0.0090752, "repeats": 7, "samples_ms": [ 0.009125, 0.008959, 0.0085, 0.007667, 0.008333, 0.007875, 0.007584 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.079917, "p95_ms": 0.08629629999999999, "repeats": 7, "samples_ms": [ 0.079917, 0.080709, 0.078792, 0.078167, 0.082125, 0.078292, 0.088084 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 0.0005, "rtol": 0.005 }, "variant": "wide", "warnings": [ { "backend": "reference", "message": "underflow encountered in multiply" } ] }, { "artifacts": { "bundle": "observations/31289ee17a4550c488dcb416ffa1110304895f4c61e977a7e3c9d949a2d8c526/reproducer.zip", "bundle_sha256": "2fac70272ac496947d1db5022cf571e340d95637677fbbb251971f92095f0269", "case": "artifacts/10dbe9dccca062e40c515ac3b06b1c9e086a6eb7eef5761c708b7c75d41eb5be/case.json", "inputs": "artifacts/10dbe9dccca062e40c515ac3b06b1c9e086a6eb7eef5761c708b7c75d41eb5be/inputs.npz", "manifest": "artifacts/10dbe9dccca062e40c515ac3b06b1c9e086a6eb7eef5761c708b7c75d41eb5be/manifest.json", "model": "artifacts/10dbe9dccca062e40c515ac3b06b1c9e086a6eb7eef5761c708b7c75d41eb5be/model.onnx", "outputs": "observations/31289ee17a4550c488dcb416ffa1110304895f4c61e977a7e3c9d949a2d8c526/outputs.npz" }, "candidate_group": "9fca66aae60d73e009d8c9e7aa8b65196b60ae7806561dd919865ff15f17ce7f", "case_id": "10dbe9dccca062e40c515ac3b06b1c9e086a6eb7eef5761c708b7c75d41eb5be", "category": "metrics", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0009765625, "max_budget_ratio": 0.1775568181818182, "max_rel": 0.0009765625, "max_ulp": 1, "mismatched": 0, "name": "cosine_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "backend": "reference", "name": "cosine_range", "passed": true }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 8, 1 ], "backend": "reference", "elements": 8, "expected_dtype": "float16", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "cosine_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "backend": "ort", "name": "cosine_range", "passed": true }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 8, 1 ], "backend": "ort", "elements": 8, "expected_dtype": "float16", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } } ], "comparison": { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0009765625, "max_budget_ratio": 0.17739932588256163, "max_rel": 0.000975609756097561, "max_ulp": 1, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, "comparison_key": "880564f59299f5e7ce4c826846d83e1540bdb453e7ee97eaf632ae89a2a48600", "contract": "cosine", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "target": { "dtype": "float16", "shape": [ 4, 257 ] }, "x": { "dtype": "float16", "shape": [ 4, 257 ] } }, "ir_version": 10, "name": "CosineSimilarity/wide/float16/4x257/seed-20260906", "observation_id": "31289ee17a4550c488dcb416ffa1110304895f4c61e977a7e3c9d949a2d8c526", "operation": "CosineSimilarity", "operators": [ "Div", "Max", "Mul", "ReduceSum", "Sqrt" ], "opset": 19, "optimization": "disabled", "output_hash": "43ef0964bdc7e3a00a5c45a014c64d80a416ef57052e53ca39cd4cec748cf27c", "outputs": {}, "parameters": { "epsilon": 1e-06 }, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.007708, "p95_ms": 0.0087419, "repeats": 7, "samples_ms": [ 0.008625, 0.008792, 0.008042, 0.007708, 0.007542, 0.007292, 0.007083 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.079292, "p95_ms": 0.0810207, "repeats": 7, "samples_ms": [ 0.079292, 0.079708, 0.080292, 0.077625, 0.078125, 0.081333, 0.078791 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 0.0005, "rtol": 0.005 }, "variant": "wide", "warnings": [ { "backend": "reference", "message": "underflow encountered in multiply" } ] }, { "artifacts": { "bundle": "observations/d7ed6c84d0522b1a5c48db7d96a68a595aa71bea8703fb124f6d7edec8506e9e/reproducer.zip", "bundle_sha256": "93771759fc1cb889772c79208ce0f27997f3dacf84f11b57cab81964a0dd6bab", "case": "artifacts/8e040b385014e899bc51ce1f3d3f2d769338eb72fbf3161cf1ef72ec269d6f25/case.json", "inputs": "artifacts/8e040b385014e899bc51ce1f3d3f2d769338eb72fbf3161cf1ef72ec269d6f25/inputs.npz", "manifest": "artifacts/8e040b385014e899bc51ce1f3d3f2d769338eb72fbf3161cf1ef72ec269d6f25/manifest.json", "model": "artifacts/8e040b385014e899bc51ce1f3d3f2d769338eb72fbf3161cf1ef72ec269d6f25/model.onnx", "outputs": "observations/d7ed6c84d0522b1a5c48db7d96a68a595aa71bea8703fb124f6d7edec8506e9e/outputs.npz" }, "candidate_group": "20373170b8a24bff744552fb15f4e9141826d54365013129b8805012ea47bc9d", "case_id": "8e040b385014e899bc51ce1f3d3f2d769338eb72fbf3161cf1ef72ec269d6f25", "category": "metrics", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 1.1920928955078125e-07, "max_budget_ratio": 0.010837729034946939, "max_rel": 1.192155923980747e-07, "max_ulp": 2, "mismatched": 0, "name": "cosine_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "reference", "name": "cosine_range", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 1 ], "backend": "reference", "elements": 8, "expected_dtype": "float32", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 1.1920928955078125e-07, "max_budget_ratio": 0.010837773668551596, "max_rel": 1.1921613247010522e-07, "max_ulp": 2, "mismatched": 0, "name": "cosine_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "cosine_range", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 5.960464477539063e-08, "max_budget_ratio": 0.005418864223832558, "max_rel": 5.960779264594819e-08, "max_ulp": 1, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 1.1920928955078125e-07, "max_budget_ratio": 0.010837772493978343, "max_rel": 1.1921611825762066e-07, "max_ulp": 2, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 1 ], "backend": "ort", "elements": 8, "expected_dtype": "float32", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } } ], "comparison": { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 1.1920928955078125e-07, "max_budget_ratio": 0.010837773668551596, "max_rel": 1.1921613247010522e-07, "max_ulp": 2, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, "comparison_key": "1d344582d1654d7d7e602259019fda47e1ce4089b67f61c10b02c72385960aa5", "contract": "cosine", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "target": { "dtype": "float32", "shape": [ 4, 257 ] }, "x": { "dtype": "float32", "shape": [ 4, 257 ] } }, "ir_version": 10, "name": "CosineSimilarity/wide/float32/4x257/seed-20260906", "observation_id": "d7ed6c84d0522b1a5c48db7d96a68a595aa71bea8703fb124f6d7edec8506e9e", "operation": "CosineSimilarity", "operators": [ "Div", "Max", "Mul", "ReduceSum", "Sqrt" ], "opset": 19, "optimization": "all", "output_hash": "3e1284d0ee201a929b69fcb3df529bb1f2a1eb62769eeaaf80a4b7cfc35bcbc3", "outputs": {}, "parameters": { "epsilon": 1e-06 }, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.006084, "p95_ms": 0.007142, "repeats": 7, "samples_ms": [ 0.007292, 0.006792, 0.006375, 0.006084, 0.005792, 0.005583, 0.005417 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.061958, "p95_ms": 0.0633919, "repeats": 7, "samples_ms": [ 0.063667, 0.06275, 0.062666, 0.061958, 0.061875, 0.061958, 0.061708 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-06, "rtol": 1e-05 }, "variant": "wide", "warnings": [] }, { "artifacts": { "bundle": "observations/4a3e17685962c43ba6304d92b6215b4f3f4081d5d764198a7c3f4924240f66ca/reproducer.zip", "bundle_sha256": "71d802eb3435b464a1b0c9062b7702ac02f3ff33d13ea8531d48d9cd7767ec3d", "case": "artifacts/8e040b385014e899bc51ce1f3d3f2d769338eb72fbf3161cf1ef72ec269d6f25/case.json", "inputs": "artifacts/8e040b385014e899bc51ce1f3d3f2d769338eb72fbf3161cf1ef72ec269d6f25/inputs.npz", "manifest": "artifacts/8e040b385014e899bc51ce1f3d3f2d769338eb72fbf3161cf1ef72ec269d6f25/manifest.json", "model": "artifacts/8e040b385014e899bc51ce1f3d3f2d769338eb72fbf3161cf1ef72ec269d6f25/model.onnx", "outputs": "observations/4a3e17685962c43ba6304d92b6215b4f3f4081d5d764198a7c3f4924240f66ca/outputs.npz" }, "candidate_group": "20373170b8a24bff744552fb15f4e9141826d54365013129b8805012ea47bc9d", "case_id": "8e040b385014e899bc51ce1f3d3f2d769338eb72fbf3161cf1ef72ec269d6f25", "category": "metrics", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 1.1920928955078125e-07, "max_budget_ratio": 0.010837729034946939, "max_rel": 1.192155923980747e-07, "max_ulp": 2, "mismatched": 0, "name": "cosine_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "reference", "name": "cosine_range", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 1 ], "backend": "reference", "elements": 8, "expected_dtype": "float32", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 1.1920928955078125e-07, "max_budget_ratio": 0.010837773668551596, "max_rel": 1.1921613247010522e-07, "max_ulp": 2, "mismatched": 0, "name": "cosine_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "cosine_range", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 5.960464477539063e-08, "max_budget_ratio": 0.005418864223832558, "max_rel": 5.960779264594819e-08, "max_ulp": 1, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 1.1920928955078125e-07, "max_budget_ratio": 0.010837772493978343, "max_rel": 1.1921611825762066e-07, "max_ulp": 2, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 1 ], "backend": "ort", "elements": 8, "expected_dtype": "float32", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } } ], "comparison": { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 1.1920928955078125e-07, "max_budget_ratio": 0.010837773668551596, "max_rel": 1.1921613247010522e-07, "max_ulp": 2, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, "comparison_key": "02236ac21d3c379b05fa82a7284d0bd952b77b8096720551a95a6c2ed1e94f64", "contract": "cosine", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "target": { "dtype": "float32", "shape": [ 4, 257 ] }, "x": { "dtype": "float32", "shape": [ 4, 257 ] } }, "ir_version": 10, "name": "CosineSimilarity/wide/float32/4x257/seed-20260906", "observation_id": "4a3e17685962c43ba6304d92b6215b4f3f4081d5d764198a7c3f4924240f66ca", "operation": "CosineSimilarity", "operators": [ "Div", "Max", "Mul", "ReduceSum", "Sqrt" ], "opset": 19, "optimization": "disabled", "output_hash": "3e1284d0ee201a929b69fcb3df529bb1f2a1eb62769eeaaf80a4b7cfc35bcbc3", "outputs": {}, "parameters": { "epsilon": 1e-06 }, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.006083, "p95_ms": 0.006966699999999999, "repeats": 7, "samples_ms": [ 0.006791, 0.007042, 0.00675, 0.006083, 0.005833, 0.005917, 0.005708 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.063625, "p95_ms": 0.0683044, "repeats": 7, "samples_ms": [ 0.068542, 0.065583, 0.06775, 0.063625, 0.062542, 0.062833, 0.062709 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-06, "rtol": 1e-05 }, "variant": "wide", "warnings": [] }, { "artifacts": { "bundle": "observations/4f824e934f3476dd791743b671b2252c70684ed894db9d31f235196911be2fca/reproducer.zip", "bundle_sha256": "600a2d069157b98eae2dae53722b4c4285f1bf0b9514011f8f92ed25b5c94d53", "case": "artifacts/67184de060383202b581a70d5e61f34c747b8b98f77955212a7c5b2875fe196e/case.json", "inputs": "artifacts/67184de060383202b581a70d5e61f34c747b8b98f77955212a7c5b2875fe196e/inputs.npz", "manifest": "artifacts/67184de060383202b581a70d5e61f34c747b8b98f77955212a7c5b2875fe196e/manifest.json", "model": "artifacts/67184de060383202b581a70d5e61f34c747b8b98f77955212a7c5b2875fe196e/model.onnx", "outputs": "observations/4f824e934f3476dd791743b671b2252c70684ed894db9d31f235196911be2fca/outputs.npz" }, "candidate_group": "11be82f8ab4ac0f5e0521435d8b2766405c06d13c832dc2f4c22649062c1b584", "case_id": "67184de060383202b581a70d5e61f34c747b8b98f77955212a7c5b2875fe196e", "category": "metrics", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "cosine_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "backend": "reference", "name": "cosine_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 8, 1 ], "backend": "reference", "elements": 8, "expected_dtype": "float64", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 2.220446049250313e-16, "max_budget_ratio": 0.0002018678174866092, "max_rel": 2.2205559871577745e-16, "max_ulp": 2, "mismatched": 0, "name": "cosine_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "backend": "ort", "name": "cosine_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 3.3306690738754696e-16, "max_budget_ratio": 0.0003028029549623161, "max_rel": 3.3308488485386035e-16, "max_ulp": 3, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 3.3306690738754696e-16, "max_budget_ratio": 0.0003028029549623161, "max_rel": 3.3308488485386035e-16, "max_ulp": 3, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 8, 1 ], "backend": "ort", "elements": 8, "expected_dtype": "float64", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } } ], "comparison": { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 2.220446049250313e-16, "max_budget_ratio": 0.0002018678174866092, "max_rel": 2.2205559871577745e-16, "max_ulp": 2, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, "comparison_key": "14542c53fac410e86c4585b8831a75374eacfc51b9788241eab23e479ea6fd57", "contract": "cosine", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "target": { "dtype": "float64", "shape": [ 4, 257 ] }, "x": { "dtype": "float64", "shape": [ 4, 257 ] } }, "ir_version": 10, "name": "CosineSimilarity/wide/float64/4x257/seed-20260906", "observation_id": "4f824e934f3476dd791743b671b2252c70684ed894db9d31f235196911be2fca", "operation": "CosineSimilarity", "operators": [ "Div", "Max", "Mul", "ReduceSum", "Sqrt" ], "opset": 19, "optimization": "all", "output_hash": "5f0d555d4501d0e72bba643b08ac423c45b34cda87283917640f6fe4e7958238", "outputs": {}, "parameters": { "epsilon": 1e-06 }, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.006709, "p95_ms": 0.0072169999999999995, "repeats": 7, "samples_ms": [ 0.007042, 0.007292, 0.006833, 0.006709, 0.00625, 0.006208, 0.006334 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.064208, "p95_ms": 0.0708587, "repeats": 7, "samples_ms": [ 0.06675, 0.063834, 0.063167, 0.063459, 0.070333, 0.071084, 0.064208 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-13, "rtol": 1e-12 }, "variant": "wide", "warnings": [] }, { "artifacts": { "bundle": "observations/88a78706fd30c90c9f925fdd24a40b4ef42380ebc004d7ad311c0fecc973a1ac/reproducer.zip", "bundle_sha256": "d769241054e8c795304d5f599c765e8721b6f4643621d1ff72a7a59023537a1a", "case": "artifacts/67184de060383202b581a70d5e61f34c747b8b98f77955212a7c5b2875fe196e/case.json", "inputs": "artifacts/67184de060383202b581a70d5e61f34c747b8b98f77955212a7c5b2875fe196e/inputs.npz", "manifest": "artifacts/67184de060383202b581a70d5e61f34c747b8b98f77955212a7c5b2875fe196e/manifest.json", "model": "artifacts/67184de060383202b581a70d5e61f34c747b8b98f77955212a7c5b2875fe196e/model.onnx", "outputs": "observations/88a78706fd30c90c9f925fdd24a40b4ef42380ebc004d7ad311c0fecc973a1ac/outputs.npz" }, "candidate_group": "11be82f8ab4ac0f5e0521435d8b2766405c06d13c832dc2f4c22649062c1b584", "case_id": "67184de060383202b581a70d5e61f34c747b8b98f77955212a7c5b2875fe196e", "category": "metrics", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "cosine_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "backend": "reference", "name": "cosine_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 8, 1 ], "backend": "reference", "elements": 8, "expected_dtype": "float64", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 2.220446049250313e-16, "max_budget_ratio": 0.0002018678174866092, "max_rel": 2.2205559871577745e-16, "max_ulp": 2, "mismatched": 0, "name": "cosine_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "backend": "ort", "name": "cosine_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 3.3306690738754696e-16, "max_budget_ratio": 0.0003028029549623161, "max_rel": 3.3308488485386035e-16, "max_ulp": 3, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 3.3306690738754696e-16, "max_budget_ratio": 0.0003028029549623161, "max_rel": 3.3308488485386035e-16, "max_ulp": 3, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 8, 1 ], "backend": "ort", "elements": 8, "expected_dtype": "float64", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } } ], "comparison": { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 2.220446049250313e-16, "max_budget_ratio": 0.0002018678174866092, "max_rel": 2.2205559871577745e-16, "max_ulp": 2, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, "comparison_key": "72c7231c84df5a58cbcf4d944da57bc428793e204dc1b079de7a0c806ac96afb", "contract": "cosine", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "target": { "dtype": "float64", "shape": [ 4, 257 ] }, "x": { "dtype": "float64", "shape": [ 4, 257 ] } }, "ir_version": 10, "name": "CosineSimilarity/wide/float64/4x257/seed-20260906", "observation_id": "88a78706fd30c90c9f925fdd24a40b4ef42380ebc004d7ad311c0fecc973a1ac", "operation": "CosineSimilarity", "operators": [ "Div", "Max", "Mul", "ReduceSum", "Sqrt" ], "opset": 19, "optimization": "disabled", "output_hash": "5f0d555d4501d0e72bba643b08ac423c45b34cda87283917640f6fe4e7958238", "outputs": {}, "parameters": { "epsilon": 1e-06 }, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.006583, "p95_ms": 0.0073624, "repeats": 7, "samples_ms": [ 0.007375, 0.007333, 0.006583, 0.006709, 0.006416, 0.006583, 0.00625 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.063125, "p95_ms": 0.06428400000000001, "repeats": 7, "samples_ms": [ 0.064584, 0.063584, 0.062833, 0.062458, 0.062333, 0.063125, 0.06325 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-13, "rtol": 1e-12 }, "variant": "wide", "warnings": [] }, { "artifacts": { "bundle": "observations/ce679f1ef74155541e0f9b83ea22197056d5b46ec8ab823246800c5b28715e88/reproducer.zip", "bundle_sha256": "df73daa6b40d2ef50fb084e0d83f8b564c80a084503c2475f847b740e95e2153", "case": "artifacts/a61718328bda924a4322f0b20e3a2f65f187d612c73cc24a6b7ce4990f489a7d/case.json", "inputs": "artifacts/a61718328bda924a4322f0b20e3a2f65f187d612c73cc24a6b7ce4990f489a7d/inputs.npz", "manifest": "artifacts/a61718328bda924a4322f0b20e3a2f65f187d612c73cc24a6b7ce4990f489a7d/manifest.json", "model": "artifacts/a61718328bda924a4322f0b20e3a2f65f187d612c73cc24a6b7ce4990f489a7d/model.onnx", "outputs": "observations/ce679f1ef74155541e0f9b83ea22197056d5b46ec8ab823246800c5b28715e88/outputs.npz" }, "candidate_group": "20498e0400a5d4130690cabdbe133631c23ed45739369d619a6c742d21eb59a5", "case_id": "a61718328bda924a4322f0b20e3a2f65f187d612c73cc24a6b7ce4990f489a7d", "category": "quantization", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "dequantize_affine_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 17 ], "backend": "reference", "elements": 136, "expected_dtype": "float32", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "dequantize_affine_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 17 ], "backend": "ort", "elements": 136, "expected_dtype": "float32", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } } ], "comparison": { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, "comparison_key": "264df8a28ce0b2bf3a0515ca495d4af5fa9e38f0fe435de1212b6f1cb4372903", "contract": "dequantize", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "int8", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "DequantizeLinear/per_axis/int8/4x17/seed-20260906", "observation_id": "ce679f1ef74155541e0f9b83ea22197056d5b46ec8ab823246800c5b28715e88", "operation": "DequantizeLinear", "operators": [ "DequantizeLinear" ], "opset": 19, "optimization": "all", "output_hash": "613f4b601b92518f2386a95d400483ee49c5f452210cadf137e60d726b8ea5df", "outputs": {}, "parameters": { "per_axis": true }, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.002958, "p95_ms": 0.0033829999999999997, "repeats": 7, "samples_ms": [ 0.003, 0.003458, 0.003208, 0.002958, 0.002958, 0.00275, 0.002834 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.00775, "p95_ms": 0.0088412, "repeats": 7, "samples_ms": [ 0.009041, 0.008375, 0.00775, 0.007542, 0.007458, 0.007875, 0.007291 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-06, "rtol": 1e-05 }, "variant": "per_axis", "warnings": [] }, { "artifacts": { "bundle": "observations/ffbdaa0eb252f543cc3f18ebbcc2dc1ab3e87acfd8919e3dc4a5158149007f25/reproducer.zip", "bundle_sha256": "e77aff9ec2a9697f13312a0f74ca6a7b34a9a6bfb12c8089140a8858116f37b1", "case": "artifacts/a61718328bda924a4322f0b20e3a2f65f187d612c73cc24a6b7ce4990f489a7d/case.json", "inputs": "artifacts/a61718328bda924a4322f0b20e3a2f65f187d612c73cc24a6b7ce4990f489a7d/inputs.npz", "manifest": "artifacts/a61718328bda924a4322f0b20e3a2f65f187d612c73cc24a6b7ce4990f489a7d/manifest.json", "model": "artifacts/a61718328bda924a4322f0b20e3a2f65f187d612c73cc24a6b7ce4990f489a7d/model.onnx", "outputs": "observations/ffbdaa0eb252f543cc3f18ebbcc2dc1ab3e87acfd8919e3dc4a5158149007f25/outputs.npz" }, "candidate_group": "20498e0400a5d4130690cabdbe133631c23ed45739369d619a6c742d21eb59a5", "case_id": "a61718328bda924a4322f0b20e3a2f65f187d612c73cc24a6b7ce4990f489a7d", "category": "quantization", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "dequantize_affine_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 17 ], "backend": "reference", "elements": 136, "expected_dtype": "float32", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "dequantize_affine_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 17 ], "backend": "ort", "elements": 136, "expected_dtype": "float32", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } } ], "comparison": { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, "comparison_key": "d8300eb4a9b548e2f004dee231d79ecb70dc3f41ccb716209e6b77249036d88e", "contract": "dequantize", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "int8", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "DequantizeLinear/per_axis/int8/4x17/seed-20260906", "observation_id": "ffbdaa0eb252f543cc3f18ebbcc2dc1ab3e87acfd8919e3dc4a5158149007f25", "operation": "DequantizeLinear", "operators": [ "DequantizeLinear" ], "opset": 19, "optimization": "disabled", "output_hash": "613f4b601b92518f2386a95d400483ee49c5f452210cadf137e60d726b8ea5df", "outputs": {}, "parameters": { "per_axis": true }, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.002958, "p95_ms": 0.0033707, "repeats": 7, "samples_ms": [ 0.003167, 0.003458, 0.002958, 0.003041, 0.002791, 0.002542, 0.002667 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.007875, "p95_ms": 0.009012699999999998, "repeats": 7, "samples_ms": [ 0.00925, 0.008459, 0.007875, 0.007833, 0.007542, 0.008208, 0.0075 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-06, "rtol": 1e-05 }, "variant": "per_axis", "warnings": [] }, { "artifacts": { "bundle": "observations/0e1c9eaa9fe725a53bbeea898fbc48a73bbc3212d5f885d19c6f57ab1c31eba4/reproducer.zip", "bundle_sha256": "ad33bea338946a863cdb70a224efec480f8bdb96844de2de4ec1651526f30b46", "case": "artifacts/e9da8623ba54b01556106a4218d6d7d3eb1b5dcde99f0aebfa2ea52a800a7e43/case.json", "inputs": "artifacts/e9da8623ba54b01556106a4218d6d7d3eb1b5dcde99f0aebfa2ea52a800a7e43/inputs.npz", "manifest": "artifacts/e9da8623ba54b01556106a4218d6d7d3eb1b5dcde99f0aebfa2ea52a800a7e43/manifest.json", "model": "artifacts/e9da8623ba54b01556106a4218d6d7d3eb1b5dcde99f0aebfa2ea52a800a7e43/model.onnx", "outputs": "observations/0e1c9eaa9fe725a53bbeea898fbc48a73bbc3212d5f885d19c6f57ab1c31eba4/outputs.npz" }, "candidate_group": "c5206c84a5fbb115be571c7e6d1b1213e470a8f69a1780c486d681143c7d978e", "case_id": "e9da8623ba54b01556106a4218d6d7d3eb1b5dcde99f0aebfa2ea52a800a7e43", "category": "quantization", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "dequantize_affine_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 17 ], "backend": "reference", "elements": 136, "expected_dtype": "float32", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "dequantize_affine_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 17 ], "backend": "ort", "elements": 136, "expected_dtype": "float32", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } } ], "comparison": { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, "comparison_key": "33537e6414ced758088e5fc57453c6eb3c725ba84b184a5825c325fc5afc052a", "contract": "dequantize", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "uint8", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "DequantizeLinear/per_axis/uint8/4x17/seed-20260906", "observation_id": "0e1c9eaa9fe725a53bbeea898fbc48a73bbc3212d5f885d19c6f57ab1c31eba4", "operation": "DequantizeLinear", "operators": [ "DequantizeLinear" ], "opset": 19, "optimization": "all", "output_hash": "03dea1b5dc3b3708d1b3069c8b799b1338f017b39a4ba33fa2518619e2ce98c0", "outputs": {}, "parameters": { "per_axis": true }, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.002834, "p95_ms": 0.003217, "repeats": 7, "samples_ms": [ 0.003042, 0.003292, 0.002916, 0.002834, 0.002625, 0.002583, 0.002834 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.007667, "p95_ms": 0.0087788, "repeats": 7, "samples_ms": [ 0.009041, 0.008167, 0.007667, 0.0075, 0.007459, 0.007375, 0.008 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-06, "rtol": 1e-05 }, "variant": "per_axis", "warnings": [] }, { "artifacts": { "bundle": "observations/a5d0bdd854d86a10ef1f6993b87d5e793b864f7311a3828951518a74cdf7cc34/reproducer.zip", "bundle_sha256": "74ccba1db4e97fd35c62ba1d0f68822cec29a15921399cc331516d6cc0f1bb66", "case": "artifacts/e9da8623ba54b01556106a4218d6d7d3eb1b5dcde99f0aebfa2ea52a800a7e43/case.json", "inputs": "artifacts/e9da8623ba54b01556106a4218d6d7d3eb1b5dcde99f0aebfa2ea52a800a7e43/inputs.npz", "manifest": "artifacts/e9da8623ba54b01556106a4218d6d7d3eb1b5dcde99f0aebfa2ea52a800a7e43/manifest.json", "model": "artifacts/e9da8623ba54b01556106a4218d6d7d3eb1b5dcde99f0aebfa2ea52a800a7e43/model.onnx", "outputs": "observations/a5d0bdd854d86a10ef1f6993b87d5e793b864f7311a3828951518a74cdf7cc34/outputs.npz" }, "candidate_group": "c5206c84a5fbb115be571c7e6d1b1213e470a8f69a1780c486d681143c7d978e", "case_id": "e9da8623ba54b01556106a4218d6d7d3eb1b5dcde99f0aebfa2ea52a800a7e43", "category": "quantization", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "dequantize_affine_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 17 ], "backend": "reference", "elements": 136, "expected_dtype": "float32", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "dequantize_affine_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 17 ], "backend": "ort", "elements": 136, "expected_dtype": "float32", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } } ], "comparison": { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, "comparison_key": "1e66d9c9a6e1e3c0fd179160459671ecfc4f635941a69d3a1c5334c7ce57a136", "contract": "dequantize", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "uint8", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "DequantizeLinear/per_axis/uint8/4x17/seed-20260906", "observation_id": "a5d0bdd854d86a10ef1f6993b87d5e793b864f7311a3828951518a74cdf7cc34", "operation": "DequantizeLinear", "operators": [ "DequantizeLinear" ], "opset": 19, "optimization": "disabled", "output_hash": "03dea1b5dc3b3708d1b3069c8b799b1338f017b39a4ba33fa2518619e2ce98c0", "outputs": {}, "parameters": { "per_axis": true }, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.003083, "p95_ms": 0.0034331, "repeats": 7, "samples_ms": [ 0.003083, 0.003458, 0.003375, 0.003125, 0.002791, 0.00275, 0.002541 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.007875, "p95_ms": 0.0084333, "repeats": 7, "samples_ms": [ 0.008583, 0.008, 0.008084, 0.00775, 0.0075, 0.007875, 0.007334 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-06, "rtol": 1e-05 }, "variant": "per_axis", "warnings": [] }, { "artifacts": { "bundle": "observations/1d5830b1b447fb818138fb3d0c983acbd583a094bd58cee57553c3fd1f57cac4/reproducer.zip", "bundle_sha256": "1446260b85fcfef1a3e012d61abfd10cfe68076b77f58fd67466d62507885e9b", "case": "artifacts/97a5c73b4e275fc3992416c386e7a9f01eda2b5ba565860b83458c142ded873c/case.json", "inputs": "artifacts/97a5c73b4e275fc3992416c386e7a9f01eda2b5ba565860b83458c142ded873c/inputs.npz", "manifest": "artifacts/97a5c73b4e275fc3992416c386e7a9f01eda2b5ba565860b83458c142ded873c/manifest.json", "model": "artifacts/97a5c73b4e275fc3992416c386e7a9f01eda2b5ba565860b83458c142ded873c/model.onnx", "outputs": "observations/1d5830b1b447fb818138fb3d0c983acbd583a094bd58cee57553c3fd1f57cac4/outputs.npz" }, "candidate_group": "26b65244d569a772483fde54981ee45fa11c1ecd5054cac9a8d9de7620fd9cf4", "case_id": "97a5c73b4e275fc3992416c386e7a9f01eda2b5ba565860b83458c142ded873c", "category": "quantization", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "dequantize_affine_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 17 ], "backend": "reference", "elements": 136, "expected_dtype": "float32", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "dequantize_affine_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 17 ], "backend": "ort", "elements": 136, "expected_dtype": "float32", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } } ], "comparison": { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, "comparison_key": "ab7169c5f6fc2c4e448d845528db4d92ad3fd789994cd22d582f3c6d7eb1f588", "contract": "dequantize", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "int8", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "DequantizeLinear/per_tensor/int8/4x17/seed-20260906", "observation_id": "1d5830b1b447fb818138fb3d0c983acbd583a094bd58cee57553c3fd1f57cac4", "operation": "DequantizeLinear", "operators": [ "DequantizeLinear" ], "opset": 19, "optimization": "all", "output_hash": "ee5371fd9eccca6b7e60783ce0ed38ef5fd7c8e06fe89310dbd24ecadaf4c7ce", "outputs": {}, "parameters": { "per_axis": false }, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.003, "p95_ms": 0.0033214, "repeats": 7, "samples_ms": [ 0.003, 0.003292, 0.003334, 0.003125, 0.002917, 0.002708, 0.00275 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.006416, "p95_ms": 0.0072248, "repeats": 7, "samples_ms": [ 0.007166, 0.00725, 0.006416, 0.006166, 0.006042, 0.006542, 0.006125 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-06, "rtol": 1e-05 }, "variant": "per_tensor", "warnings": [] }, { "artifacts": { "bundle": "observations/80baa9543eb515d4f8500ff93cffdc4162a5fbb8e41c5cc171f99a17c48113b0/reproducer.zip", "bundle_sha256": "002d8b11d73621c101302aa69efc81bc707301e4bfd69f9e8e33c7a7b510e52d", "case": "artifacts/97a5c73b4e275fc3992416c386e7a9f01eda2b5ba565860b83458c142ded873c/case.json", "inputs": "artifacts/97a5c73b4e275fc3992416c386e7a9f01eda2b5ba565860b83458c142ded873c/inputs.npz", "manifest": "artifacts/97a5c73b4e275fc3992416c386e7a9f01eda2b5ba565860b83458c142ded873c/manifest.json", "model": "artifacts/97a5c73b4e275fc3992416c386e7a9f01eda2b5ba565860b83458c142ded873c/model.onnx", "outputs": "observations/80baa9543eb515d4f8500ff93cffdc4162a5fbb8e41c5cc171f99a17c48113b0/outputs.npz" }, "candidate_group": "26b65244d569a772483fde54981ee45fa11c1ecd5054cac9a8d9de7620fd9cf4", "case_id": "97a5c73b4e275fc3992416c386e7a9f01eda2b5ba565860b83458c142ded873c", "category": "quantization", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "dequantize_affine_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 17 ], "backend": "reference", "elements": 136, "expected_dtype": "float32", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "dequantize_affine_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 17 ], "backend": "ort", "elements": 136, "expected_dtype": "float32", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } } ], "comparison": { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, "comparison_key": "f24fe44e092e1b2d377cc56cf9e265bddf7ce7b4200ff673d2042c3bf0fdfd4e", "contract": "dequantize", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "int8", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "DequantizeLinear/per_tensor/int8/4x17/seed-20260906", "observation_id": "80baa9543eb515d4f8500ff93cffdc4162a5fbb8e41c5cc171f99a17c48113b0", "operation": "DequantizeLinear", "operators": [ "DequantizeLinear" ], "opset": 19, "optimization": "disabled", "output_hash": "ee5371fd9eccca6b7e60783ce0ed38ef5fd7c8e06fe89310dbd24ecadaf4c7ce", "outputs": {}, "parameters": { "per_axis": false }, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.002917, "p95_ms": 0.0032706, "repeats": 7, "samples_ms": [ 0.002958, 0.003333, 0.002917, 0.003125, 0.00275, 0.002584, 0.0025 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.006542, "p95_ms": 0.0076124999999999995, "repeats": 7, "samples_ms": [ 0.007875, 0.007, 0.006875, 0.006542, 0.006417, 0.006291, 0.00625 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-06, "rtol": 1e-05 }, "variant": "per_tensor", "warnings": [] }, { "artifacts": { "bundle": "observations/4ac3573be87be4a9bace667356a02c02b161f409756bae543f14c769a1996dd9/reproducer.zip", "bundle_sha256": "820bedf8585f011a1d551b5c46a186eb39d3781b730094ac3beda45fe3b92447", "case": "artifacts/f1a4610c338b38a8001ffc15da632dc2520e01e8c4d2b6d5b7dd8332030b92cd/case.json", "inputs": "artifacts/f1a4610c338b38a8001ffc15da632dc2520e01e8c4d2b6d5b7dd8332030b92cd/inputs.npz", "manifest": "artifacts/f1a4610c338b38a8001ffc15da632dc2520e01e8c4d2b6d5b7dd8332030b92cd/manifest.json", "model": "artifacts/f1a4610c338b38a8001ffc15da632dc2520e01e8c4d2b6d5b7dd8332030b92cd/model.onnx", "outputs": "observations/4ac3573be87be4a9bace667356a02c02b161f409756bae543f14c769a1996dd9/outputs.npz" }, "candidate_group": "49d0d8d960417b77499e2527d0bf0dbd0d358d56cce8e1dcbab1de8ffe075592", "case_id": "f1a4610c338b38a8001ffc15da632dc2520e01e8c4d2b6d5b7dd8332030b92cd", "category": "quantization", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "dequantize_affine_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 17 ], "backend": "reference", "elements": 136, "expected_dtype": "float32", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "dequantize_affine_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 17 ], "backend": "ort", "elements": 136, "expected_dtype": "float32", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } } ], "comparison": { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, "comparison_key": "31787ecfb6b09e1f0eb0e5869998281575d0263db9e79bca40ba8323da2fba0c", "contract": "dequantize", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "uint8", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "DequantizeLinear/per_tensor/uint8/4x17/seed-20260906", "observation_id": "4ac3573be87be4a9bace667356a02c02b161f409756bae543f14c769a1996dd9", "operation": "DequantizeLinear", "operators": [ "DequantizeLinear" ], "opset": 19, "optimization": "all", "output_hash": "3e7ff6072758d6eaa7375e2dd7968e948daa90406f946487d698ab62e75644c1", "outputs": {}, "parameters": { "per_axis": false }, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.002792, "p95_ms": 0.0033956, "repeats": 7, "samples_ms": [ 0.00325, 0.003458, 0.002958, 0.002791, 0.002666, 0.002625, 0.002792 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.006, "p95_ms": 0.0068373999999999996, "repeats": 7, "samples_ms": [ 0.007, 0.006458, 0.006125, 0.006, 0.005875, 0.005875, 0.005791 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-06, "rtol": 1e-05 }, "variant": "per_tensor", "warnings": [] }, { "artifacts": { "bundle": "observations/3961b0ff41f39e77f66e868bf92c2a79ed1db30a767e23f185f73ab971598251/reproducer.zip", "bundle_sha256": "c0654334d0381e570687b23d64b1b44e338479ca4a54a15d6b61cc3f226abaa6", "case": "artifacts/f1a4610c338b38a8001ffc15da632dc2520e01e8c4d2b6d5b7dd8332030b92cd/case.json", "inputs": "artifacts/f1a4610c338b38a8001ffc15da632dc2520e01e8c4d2b6d5b7dd8332030b92cd/inputs.npz", "manifest": "artifacts/f1a4610c338b38a8001ffc15da632dc2520e01e8c4d2b6d5b7dd8332030b92cd/manifest.json", "model": "artifacts/f1a4610c338b38a8001ffc15da632dc2520e01e8c4d2b6d5b7dd8332030b92cd/model.onnx", "outputs": "observations/3961b0ff41f39e77f66e868bf92c2a79ed1db30a767e23f185f73ab971598251/outputs.npz" }, "candidate_group": "49d0d8d960417b77499e2527d0bf0dbd0d358d56cce8e1dcbab1de8ffe075592", "case_id": "f1a4610c338b38a8001ffc15da632dc2520e01e8c4d2b6d5b7dd8332030b92cd", "category": "quantization", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "dequantize_affine_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 17 ], "backend": "reference", "elements": 136, "expected_dtype": "float32", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "dequantize_affine_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 17 ], "backend": "ort", "elements": 136, "expected_dtype": "float32", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } } ], "comparison": { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, "comparison_key": "e4b86ef56e0ad008dc1595d7be90c7eee083637419f7304f0bc795f8b0759544", "contract": "dequantize", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "uint8", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "DequantizeLinear/per_tensor/uint8/4x17/seed-20260906", "observation_id": "3961b0ff41f39e77f66e868bf92c2a79ed1db30a767e23f185f73ab971598251", "operation": "DequantizeLinear", "operators": [ "DequantizeLinear" ], "opset": 19, "optimization": "disabled", "output_hash": "3e7ff6072758d6eaa7375e2dd7968e948daa90406f946487d698ab62e75644c1", "outputs": {}, "parameters": { "per_axis": false }, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.002959, "p95_ms": 0.0041626, "repeats": 7, "samples_ms": [ 0.004375, 0.003667, 0.003458, 0.002917, 0.002959, 0.00275, 0.002875 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.006, "p95_ms": 0.0066536, "repeats": 7, "samples_ms": [ 0.006791, 0.006333, 0.006, 0.005834, 0.005792, 0.006, 0.005833 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-06, "rtol": 1e-05 }, "variant": "per_tensor", "warnings": [] }, { "artifacts": { "bundle": "observations/a0ca68f92de4e488c2b09aafd0cdf17651b9c79fcc1a5bb3035bc7832eb43349/reproducer.zip", "bundle_sha256": "604f399f6975f7e1f079c9420a701d634731b26d7ceecf525cd72b011bc6ce29", "case": "artifacts/46cb4f247955ce36b18941bf85367f94efe9d432845073f977f4222d55af7b2b/case.json", "inputs": "artifacts/46cb4f247955ce36b18941bf85367f94efe9d432845073f977f4222d55af7b2b/inputs.npz", "manifest": "artifacts/46cb4f247955ce36b18941bf85367f94efe9d432845073f977f4222d55af7b2b/manifest.json", "model": "artifacts/46cb4f247955ce36b18941bf85367f94efe9d432845073f977f4222d55af7b2b/model.onnx", "outputs": "observations/a0ca68f92de4e488c2b09aafd0cdf17651b9c79fcc1a5bb3035bc7832eb43349/outputs.npz" }, "candidate_group": "8f880e862fdb48436f09a6af77ce8580329ea22b1626aea760766f8bc06829da", "case_id": "46cb4f247955ce36b18941bf85367f94efe9d432845073f977f4222d55af7b2b", "category": "normalization", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "centered_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "normalized_variance", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 17 ], "backend": "reference", "elements": 136, "expected_dtype": "float32", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "centered_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "normalized_variance", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 17 ], "backend": "ort", "elements": 136, "expected_dtype": "float32", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } } ], "comparison": { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, "comparison_key": "7905ea452d021065e06bbac8c2c6bb305527953a5d1ea558044b27d2636d6f75", "contract": "layer_norm", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "float32", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "LayerNormalization/constant/float32/4x17/seed-20260906", "observation_id": "a0ca68f92de4e488c2b09aafd0cdf17651b9c79fcc1a5bb3035bc7832eb43349", "operation": "LayerNormalization", "operators": [ "LayerNormalization" ], "opset": 19, "optimization": "all", "output_hash": "e86796be31db15631117eb08a7c06a248c896ec4da8ba006ecad1231eeda523b", "outputs": {}, "parameters": { "epsilon": 1e-05 }, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.002875, "p95_ms": 0.0033499999999999997, "repeats": 7, "samples_ms": [ 0.002916, 0.0035, 0.002875, 0.003, 0.002625, 0.002625, 0.0025 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.0165, "p95_ms": 0.0176126, "repeats": 7, "samples_ms": [ 0.01775, 0.017292, 0.0165, 0.016666, 0.016459, 0.0165, 0.015959 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-06, "rtol": 1e-05 }, "variant": "constant", "warnings": [] }, { "artifacts": { "bundle": "observations/ca42778decd926c61aab7f3b3464a41568b243c928da99bd6903112eb5aff0fb/reproducer.zip", "bundle_sha256": "d1863558c1bf3dadc91fe6a3b0349b321517671cb84f04fb3caa545e155beb56", "case": "artifacts/46cb4f247955ce36b18941bf85367f94efe9d432845073f977f4222d55af7b2b/case.json", "inputs": "artifacts/46cb4f247955ce36b18941bf85367f94efe9d432845073f977f4222d55af7b2b/inputs.npz", "manifest": "artifacts/46cb4f247955ce36b18941bf85367f94efe9d432845073f977f4222d55af7b2b/manifest.json", "model": "artifacts/46cb4f247955ce36b18941bf85367f94efe9d432845073f977f4222d55af7b2b/model.onnx", "outputs": "observations/ca42778decd926c61aab7f3b3464a41568b243c928da99bd6903112eb5aff0fb/outputs.npz" }, "candidate_group": "8f880e862fdb48436f09a6af77ce8580329ea22b1626aea760766f8bc06829da", "case_id": "46cb4f247955ce36b18941bf85367f94efe9d432845073f977f4222d55af7b2b", "category": "normalization", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "centered_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "normalized_variance", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 17 ], "backend": "reference", "elements": 136, "expected_dtype": "float32", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "centered_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "normalized_variance", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 17 ], "backend": "ort", "elements": 136, "expected_dtype": "float32", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } } ], "comparison": { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, "comparison_key": "6683036f91b20abbbdadc15fa03cecdc53c899e17a435ce8971a7745b18b7408", "contract": "layer_norm", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "float32", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "LayerNormalization/constant/float32/4x17/seed-20260906", "observation_id": "ca42778decd926c61aab7f3b3464a41568b243c928da99bd6903112eb5aff0fb", "operation": "LayerNormalization", "operators": [ "LayerNormalization" ], "opset": 19, "optimization": "disabled", "output_hash": "e86796be31db15631117eb08a7c06a248c896ec4da8ba006ecad1231eeda523b", "outputs": {}, "parameters": { "epsilon": 1e-05 }, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.003083, "p95_ms": 0.0042499, "repeats": 7, "samples_ms": [ 0.003958, 0.004375, 0.003833, 0.003042, 0.003, 0.00275, 0.003083 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.016292, "p95_ms": 0.0174288, "repeats": 7, "samples_ms": [ 0.017541, 0.017167, 0.0165, 0.016125, 0.016208, 0.016292, 0.016125 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-06, "rtol": 1e-05 }, "variant": "constant", "warnings": [] }, { "artifacts": { "bundle": "observations/8206d24d72ae070f3bb72dcacf4d0ebdae45cb31291fed086a39c4b2e92b0ac5/reproducer.zip", "bundle_sha256": "23301696a474233bdb3a48077e60a164567dc0581d8d9a702abad8b0a2548b5a", "case": "artifacts/2438080ccfb7d623c6113b3157ecf20fe7b1833aad624dee6097ad00d8e5f49b/case.json", "inputs": "artifacts/2438080ccfb7d623c6113b3157ecf20fe7b1833aad624dee6097ad00d8e5f49b/inputs.npz", "manifest": "artifacts/2438080ccfb7d623c6113b3157ecf20fe7b1833aad624dee6097ad00d8e5f49b/manifest.json", "model": "artifacts/2438080ccfb7d623c6113b3157ecf20fe7b1833aad624dee6097ad00d8e5f49b/model.onnx", "outputs": "observations/8206d24d72ae070f3bb72dcacf4d0ebdae45cb31291fed086a39c4b2e92b0ac5/outputs.npz" }, "candidate_group": "1e80c7f0088bf30f3f25ba6fd4c12c7f430e76f40e13d89b64a23712f2176a64", "case_id": "2438080ccfb7d623c6113b3157ecf20fe7b1833aad624dee6097ad00d8e5f49b", "category": "normalization", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": [ 0, 0 ], "max_abs": 0.003606557846069336, "max_budget_ratio": 1922.162731632352, "max_rel": 0.15000081518967973, "max_ulp": 1819600, "mismatched": 68, "name": "centered_float64_oracle", "nonfinite": 0, "numerical_divergence": true, "passed": false, "reason": "tolerance_exceeded", "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": [ 1, 0 ], "max_abs": 1.2926066931706615e-05, "max_budget_ratio": 1.1762619729002752, "max_rel": 1.294016457654872e-05, "max_ulp": 116427660434, "mismatched": 2, "name": "normalized_variance", "nonfinite": 0, "numerical_divergence": true, "passed": false, "reason": "tolerance_exceeded", "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 17 ], "backend": "reference", "elements": 136, "expected_dtype": "float32", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 1.1920928955078125e-07, "max_budget_ratio": 0.01044786027717045, "max_rel": 1.1451504538746566e-07, "max_ulp": 1, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "finite_output", "passed": false }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": [ 0, 0 ], "max_abs": 2.3035069555044174, "max_budget_ratio": 92825.62746395587, "max_rel": 0.9683745448090036, "max_ulp": 41785349, "mismatched": 68, "name": "centered_float64_oracle", "nonfinite": 17, "numerical_divergence": true, "passed": false, "reason": "nonfinite_output", "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": [ 0, 0 ], "max_abs": 0.9978925512795997, "max_budget_ratio": 90805.90729743437, "max_rel": 0.9989623274188263, "max_ulp": 44749654716833871, "mismatched": 4, "name": "normalized_variance", "nonfinite": 1, "numerical_divergence": true, "passed": false, "reason": "nonfinite_output", "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": [ 0, 0 ], "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 17, "name": "batch_partition", "nonfinite": 17, "numerical_divergence": false, "passed": false, "reason": "nonfinite_output", "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": [ 3, 0 ], "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 17, "name": "batch_permutation", "nonfinite": 17, "numerical_divergence": false, "passed": false, "reason": "nonfinite_output", "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": [ 0, 0 ], "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 17, "name": "batch_reshape", "nonfinite": 17, "numerical_divergence": false, "passed": false, "reason": "nonfinite_output", "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 17 ], "backend": "ort", "elements": 136, "expected_dtype": "float32", "expected_shape": [ 8, 17 ], "first_mismatch": [ 0, 0 ], "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 34, "name": "batch_duplicate", "nonfinite": 34, "numerical_divergence": false, "passed": false, "reason": "nonfinite_output", "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": [ 0, 0 ], "max_abs": 2.307088240981102, "max_budget_ratio": 1295869.857260357, "max_rel": 30.043243140148853, "max_ulp": 41660865, "mismatched": 68, "name": "equivalent_graph", "nonfinite": 17, "numerical_divergence": true, "passed": false, "reason": "nonfinite_output", "tolerance": { "atol": 1e-06, "rtol": 1e-05 } } ], "comparison": { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": [ 0, 0 ], "max_abs": 2.307088240981102, "max_budget_ratio": 92836.00835136107, "max_rel": 0.9677868706155122, "max_ulp": 41660865, "mismatched": 68, "nonfinite": 17, "numerical_divergence": true, "passed": false, "reason": "nonfinite_output", "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, "comparison_key": "ad556c198ece166429c2d9a45e3f5fcd07be1f8939e4cbeecf761d01f101c256", "contract": "layer_norm", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "float32", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "LayerNormalization/large_offset/float32/4x17/seed-20260906", "observation_id": "8206d24d72ae070f3bb72dcacf4d0ebdae45cb31291fed086a39c4b2e92b0ac5", "operation": "LayerNormalization", "operators": [ "LayerNormalization" ], "opset": 19, "optimization": "all", "output_hash": "db1cf15ac434c6373538a34dcbee009da252dceffb10a0acc54f648714fec65e", "outputs": {}, "parameters": { "epsilon": 1e-05 }, "regression": "not_assessed", "seed": 20260906, "status": "divergence", "timings": { "ort": { "median_ms": 0.002959, "p95_ms": 0.0034455999999999996, "repeats": 7, "samples_ms": [ 0.003125, 0.003583, 0.002959, 0.003125, 0.002833, 0.002917, 0.002667 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.01625, "p95_ms": 0.0179713, "repeats": 7, "samples_ms": [ 0.018334, 0.017125, 0.016542, 0.01625, 0.015958, 0.016042, 0.015958 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-06, "rtol": 1e-05 }, "variant": "large_offset", "warnings": [] }, { "artifacts": { "bundle": "observations/575b366c04dcfa62a4fa54c49aa68e6df2691ca603fcc7fdc6492dc98d66fe2c/reproducer.zip", "bundle_sha256": "946f1ddaa46275045342d6a840b71d96ca9010017a7f2f98309738a14527f01b", "case": "artifacts/2438080ccfb7d623c6113b3157ecf20fe7b1833aad624dee6097ad00d8e5f49b/case.json", "inputs": "artifacts/2438080ccfb7d623c6113b3157ecf20fe7b1833aad624dee6097ad00d8e5f49b/inputs.npz", "manifest": "artifacts/2438080ccfb7d623c6113b3157ecf20fe7b1833aad624dee6097ad00d8e5f49b/manifest.json", "model": "artifacts/2438080ccfb7d623c6113b3157ecf20fe7b1833aad624dee6097ad00d8e5f49b/model.onnx", "outputs": "observations/575b366c04dcfa62a4fa54c49aa68e6df2691ca603fcc7fdc6492dc98d66fe2c/outputs.npz" }, "candidate_group": "1e80c7f0088bf30f3f25ba6fd4c12c7f430e76f40e13d89b64a23712f2176a64", "case_id": "2438080ccfb7d623c6113b3157ecf20fe7b1833aad624dee6097ad00d8e5f49b", "category": "normalization", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": [ 0, 0 ], "max_abs": 0.003606557846069336, "max_budget_ratio": 1922.162731632352, "max_rel": 0.15000081518967973, "max_ulp": 1819600, "mismatched": 68, "name": "centered_float64_oracle", "nonfinite": 0, "numerical_divergence": true, "passed": false, "reason": "tolerance_exceeded", "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": [ 1, 0 ], "max_abs": 1.2926066931706615e-05, "max_budget_ratio": 1.1762619729002752, "max_rel": 1.294016457654872e-05, "max_ulp": 116427660434, "mismatched": 2, "name": "normalized_variance", "nonfinite": 0, "numerical_divergence": true, "passed": false, "reason": "tolerance_exceeded", "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 17 ], "backend": "reference", "elements": 136, "expected_dtype": "float32", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 1.1920928955078125e-07, "max_budget_ratio": 0.01044786027717045, "max_rel": 1.1451504538746566e-07, "max_ulp": 1, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "finite_output", "passed": false }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": [ 0, 0 ], "max_abs": 2.3035069555044174, "max_budget_ratio": 92825.62746395587, "max_rel": 0.9683745448090036, "max_ulp": 41785349, "mismatched": 68, "name": "centered_float64_oracle", "nonfinite": 17, "numerical_divergence": true, "passed": false, "reason": "nonfinite_output", "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": [ 0, 0 ], "max_abs": 0.9978925512795997, "max_budget_ratio": 90805.90729743437, "max_rel": 0.9989623274188263, "max_ulp": 44749654716833871, "mismatched": 4, "name": "normalized_variance", "nonfinite": 1, "numerical_divergence": true, "passed": false, "reason": "nonfinite_output", "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": [ 0, 0 ], "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 17, "name": "batch_partition", "nonfinite": 17, "numerical_divergence": false, "passed": false, "reason": "nonfinite_output", "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": [ 3, 0 ], "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 17, "name": "batch_permutation", "nonfinite": 17, "numerical_divergence": false, "passed": false, "reason": "nonfinite_output", "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": [ 0, 0 ], "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 17, "name": "batch_reshape", "nonfinite": 17, "numerical_divergence": false, "passed": false, "reason": "nonfinite_output", "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 17 ], "backend": "ort", "elements": 136, "expected_dtype": "float32", "expected_shape": [ 8, 17 ], "first_mismatch": [ 0, 0 ], "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 34, "name": "batch_duplicate", "nonfinite": 34, "numerical_divergence": false, "passed": false, "reason": "nonfinite_output", "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": [ 0, 0 ], "max_abs": 2.307088240981102, "max_budget_ratio": 1295869.857260357, "max_rel": 30.043243140148853, "max_ulp": 41660865, "mismatched": 68, "name": "equivalent_graph", "nonfinite": 17, "numerical_divergence": true, "passed": false, "reason": "nonfinite_output", "tolerance": { "atol": 1e-06, "rtol": 1e-05 } } ], "comparison": { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": [ 0, 0 ], "max_abs": 2.307088240981102, "max_budget_ratio": 92836.00835136107, "max_rel": 0.9677868706155122, "max_ulp": 41660865, "mismatched": 68, "nonfinite": 17, "numerical_divergence": true, "passed": false, "reason": "nonfinite_output", "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, "comparison_key": "20931de89d03eba9502ad33009a295d9a31dcda25f6275a01fa71bf44a0f9fca", "contract": "layer_norm", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "float32", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "LayerNormalization/large_offset/float32/4x17/seed-20260906", "observation_id": "575b366c04dcfa62a4fa54c49aa68e6df2691ca603fcc7fdc6492dc98d66fe2c", "operation": "LayerNormalization", "operators": [ "LayerNormalization" ], "opset": 19, "optimization": "disabled", "output_hash": "db1cf15ac434c6373538a34dcbee009da252dceffb10a0acc54f648714fec65e", "outputs": {}, "parameters": { "epsilon": 1e-05 }, "regression": "not_assessed", "seed": 20260906, "status": "divergence", "timings": { "ort": { "median_ms": 0.002875, "p95_ms": 0.0033206999999999998, "repeats": 7, "samples_ms": [ 0.003292, 0.003333, 0.002875, 0.003, 0.002792, 0.002625, 0.00275 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.016291, "p95_ms": 0.0182545, "repeats": 7, "samples_ms": [ 0.018667, 0.017292, 0.016917, 0.016291, 0.015959, 0.016125, 0.01575 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-06, "rtol": 1e-05 }, "variant": "large_offset", "warnings": [] }, { "artifacts": { "bundle": "observations/cd64e61d1577f171e0350dd8bf26cefad67466461444764757829df92c10f6e6/reproducer.zip", "bundle_sha256": "26407b21b4d6f8c4905fc6b7632f9c804fe3e7b0ce374d1caf6107692492f1d5", "case": "artifacts/bdb2f644c8709390a833217ff05660cf84b140fc5b1d283659abe6e9115048fa/case.json", "inputs": "artifacts/bdb2f644c8709390a833217ff05660cf84b140fc5b1d283659abe6e9115048fa/inputs.npz", "manifest": "artifacts/bdb2f644c8709390a833217ff05660cf84b140fc5b1d283659abe6e9115048fa/manifest.json", "model": "artifacts/bdb2f644c8709390a833217ff05660cf84b140fc5b1d283659abe6e9115048fa/model.onnx", "outputs": "observations/cd64e61d1577f171e0350dd8bf26cefad67466461444764757829df92c10f6e6/outputs.npz" }, "candidate_group": "49acb1fbe8f09989eb3980680471c1f864d39895dfe6e531d13595218b206743", "case_id": "bdb2f644c8709390a833217ff05660cf84b140fc5b1d283659abe6e9115048fa", "category": "normalization", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 2.384185791015625e-07, "max_budget_ratio": 0.014367390987827957, "max_rel": 9.021351488666517e-07, "max_ulp": 11, "mismatched": 0, "name": "centered_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 1.3013316613186277e-07, "max_budget_ratio": 0.01183039109310294, "max_rel": 1.3013441561444994e-07, "max_ulp": 1172135357, "mismatched": 0, "name": "normalized_variance", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 17 ], "backend": "reference", "elements": 136, "expected_dtype": "float32", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 1.1920928955078125e-07, "max_budget_ratio": 0.009722188653665286, "max_rel": 1.1479269128709698e-07, "max_ulp": 1, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 2.384185791015625e-07, "max_budget_ratio": 0.0224435035716283, "max_rel": 9.841474351272564e-07, "max_ulp": 12, "mismatched": 0, "name": "centered_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 1.42490968912945e-07, "max_budget_ratio": 0.012953837515696677, "max_rel": 1.4249233704982953e-07, "max_ulp": 1283444549, "mismatched": 0, "name": "normalized_variance", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 17 ], "backend": "ort", "elements": 136, "expected_dtype": "float32", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 2.384185791015625e-07, "max_budget_ratio": 0.02394564992619735, "max_rel": 1.8862807276155682e-06, "max_ulp": 23, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } } ], "comparison": { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 2.384185791015625e-07, "max_budget_ratio": 0.021551089578071236, "max_rel": 1.886284285677263e-06, "max_ulp": 23, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, "comparison_key": "b0940dc505ae782b6ffa161b94c1c7c3e5c992f50fd89a964daf1d682cf55fd4", "contract": "layer_norm", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "float32", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "LayerNormalization/ordinary/float32/4x17/seed-20260906", "observation_id": "cd64e61d1577f171e0350dd8bf26cefad67466461444764757829df92c10f6e6", "operation": "LayerNormalization", "operators": [ "LayerNormalization" ], "opset": 19, "optimization": "all", "output_hash": "0a2560bbafa80401001fbfd214901095017b78facf28eb27e3ea499c6b0efe3c", "outputs": {}, "parameters": { "epsilon": 1e-05 }, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.003209, "p95_ms": 0.0036292, "repeats": 7, "samples_ms": [ 0.003541, 0.003667, 0.003292, 0.003125, 0.003167, 0.003041, 0.003209 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.0165, "p95_ms": 0.0179456, "repeats": 7, "samples_ms": [ 0.018083, 0.017625, 0.017167, 0.0165, 0.015917, 0.016083, 0.01575 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-06, "rtol": 1e-05 }, "variant": "ordinary", "warnings": [] }, { "artifacts": { "bundle": "observations/62ab26522efad2393a10219f22ac9da021a08bbe7727c776bbdf59cd98287497/reproducer.zip", "bundle_sha256": "d1a7100afe91989b1d948b5a741edd483b71537e0b0e41b34e3477875f144ed3", "case": "artifacts/bdb2f644c8709390a833217ff05660cf84b140fc5b1d283659abe6e9115048fa/case.json", "inputs": "artifacts/bdb2f644c8709390a833217ff05660cf84b140fc5b1d283659abe6e9115048fa/inputs.npz", "manifest": "artifacts/bdb2f644c8709390a833217ff05660cf84b140fc5b1d283659abe6e9115048fa/manifest.json", "model": "artifacts/bdb2f644c8709390a833217ff05660cf84b140fc5b1d283659abe6e9115048fa/model.onnx", "outputs": "observations/62ab26522efad2393a10219f22ac9da021a08bbe7727c776bbdf59cd98287497/outputs.npz" }, "candidate_group": "49acb1fbe8f09989eb3980680471c1f864d39895dfe6e531d13595218b206743", "case_id": "bdb2f644c8709390a833217ff05660cf84b140fc5b1d283659abe6e9115048fa", "category": "normalization", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 2.384185791015625e-07, "max_budget_ratio": 0.014367390987827957, "max_rel": 9.021351488666517e-07, "max_ulp": 11, "mismatched": 0, "name": "centered_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 1.3013316613186277e-07, "max_budget_ratio": 0.01183039109310294, "max_rel": 1.3013441561444994e-07, "max_ulp": 1172135357, "mismatched": 0, "name": "normalized_variance", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 17 ], "backend": "reference", "elements": 136, "expected_dtype": "float32", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 1.1920928955078125e-07, "max_budget_ratio": 0.009722188653665286, "max_rel": 1.1479269128709698e-07, "max_ulp": 1, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 2.384185791015625e-07, "max_budget_ratio": 0.0224435035716283, "max_rel": 9.841474351272564e-07, "max_ulp": 12, "mismatched": 0, "name": "centered_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 1.42490968912945e-07, "max_budget_ratio": 0.012953837515696677, "max_rel": 1.4249233704982953e-07, "max_ulp": 1283444549, "mismatched": 0, "name": "normalized_variance", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 17 ], "backend": "ort", "elements": 136, "expected_dtype": "float32", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 2.384185791015625e-07, "max_budget_ratio": 0.02394564992619735, "max_rel": 1.8862807276155682e-06, "max_ulp": 23, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } } ], "comparison": { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 2.384185791015625e-07, "max_budget_ratio": 0.021551089578071236, "max_rel": 1.886284285677263e-06, "max_ulp": 23, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, "comparison_key": "c4635de49a7049beb503199d34d6be040e4141d99b5ec3619ce0e75f09b9445e", "contract": "layer_norm", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "float32", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "LayerNormalization/ordinary/float32/4x17/seed-20260906", "observation_id": "62ab26522efad2393a10219f22ac9da021a08bbe7727c776bbdf59cd98287497", "operation": "LayerNormalization", "operators": [ "LayerNormalization" ], "opset": 19, "optimization": "disabled", "output_hash": "0a2560bbafa80401001fbfd214901095017b78facf28eb27e3ea499c6b0efe3c", "outputs": {}, "parameters": { "epsilon": 1e-05 }, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.003334, "p95_ms": 0.0037251, "repeats": 7, "samples_ms": [ 0.003667, 0.00375, 0.003416, 0.003334, 0.003333, 0.003125, 0.003041 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.015083, "p95_ms": 0.0190205, "repeats": 7, "samples_ms": [ 0.019583, 0.017708, 0.016458, 0.015083, 0.014792, 0.014875, 0.014666 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-06, "rtol": 1e-05 }, "variant": "ordinary", "warnings": [] }, { "artifacts": { "bundle": "observations/6801cd9914b32dd5ce8bbae833c8537607bf4e99e834282d36bd4719d46affd1/reproducer.zip", "bundle_sha256": "7f97e81ad5dac1e7246514f944a534737e84e2290219b5444800765150c18d90", "case": "artifacts/ea6ee1fa428648655807b50b6ee7ce84519017e6b852c34b6f2a367537033014/case.json", "inputs": "artifacts/ea6ee1fa428648655807b50b6ee7ce84519017e6b852c34b6f2a367537033014/inputs.npz", "manifest": "artifacts/ea6ee1fa428648655807b50b6ee7ce84519017e6b852c34b6f2a367537033014/manifest.json", "model": "artifacts/ea6ee1fa428648655807b50b6ee7ce84519017e6b852c34b6f2a367537033014/model.onnx", "outputs": "observations/6801cd9914b32dd5ce8bbae833c8537607bf4e99e834282d36bd4719d46affd1/outputs.npz" }, "candidate_group": "2e7f18e5d825e764ea2890c1747ece69fa26764698875d70cd8577c904a17708", "case_id": "ea6ee1fa428648655807b50b6ee7ce84519017e6b852c34b6f2a367537033014", "category": "normalization", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 83 ], "backend": "reference", "elements": 332, "expected_dtype": "float32", "expected_shape": [ 4, 83 ], "first_mismatch": null, "max_abs": 2.384185791015625e-07, "max_budget_ratio": 0.01743071600870713, "max_rel": 0.0003116543515307663, "max_ulp": 4794, "mismatched": 0, "name": "centered_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 1.8407762047978338e-07, "max_budget_ratio": 0.016738852809237422, "max_rel": 1.8413235842348725e-07, "max_ulp": 1658023806, "mismatched": 0, "name": "normalized_variance", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 83 ], "backend": "reference", "elements": 332, "expected_dtype": "float32", "expected_shape": [ 4, 83 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 83 ], "backend": "reference", "elements": 332, "expected_dtype": "float32", "expected_shape": [ 4, 83 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 83 ], "backend": "reference", "elements": 332, "expected_dtype": "float32", "expected_shape": [ 4, 83 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 83 ], "backend": "reference", "elements": 664, "expected_dtype": "float32", "expected_shape": [ 8, 83 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 83 ], "backend": "reference", "elements": 332, "expected_dtype": "float32", "expected_shape": [ 4, 83 ], "first_mismatch": null, "max_abs": 2.384185791015625e-07, "max_budget_ratio": 0.010991641252526538, "max_rel": 1.1671343373956611e-07, "max_ulp": 1, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 83 ], "backend": "ort", "elements": 332, "expected_dtype": "float32", "expected_shape": [ 4, 83 ], "first_mismatch": null, "max_abs": 2.384185791015625e-07, "max_budget_ratio": 0.01787772548103806, "max_rel": 0.00024599500754952437, "max_ulp": 3784, "mismatched": 0, "name": "centered_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 1.558245660282509e-07, "max_budget_ratio": 0.014169887807201943, "max_rel": 1.5587318724346586e-07, "max_ulp": 1403542915, "mismatched": 0, "name": "normalized_variance", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 83 ], "backend": "ort", "elements": 332, "expected_dtype": "float32", "expected_shape": [ 4, 83 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 83 ], "backend": "ort", "elements": 332, "expected_dtype": "float32", "expected_shape": [ 4, 83 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 83 ], "backend": "ort", "elements": 332, "expected_dtype": "float32", "expected_shape": [ 4, 83 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 83 ], "backend": "ort", "elements": 664, "expected_dtype": "float32", "expected_shape": [ 8, 83 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 83 ], "backend": "ort", "elements": 332, "expected_dtype": "float32", "expected_shape": [ 4, 83 ], "first_mismatch": null, "max_abs": 2.384185791015625e-07, "max_budget_ratio": 0.031192768713552702, "max_rel": 0.0005578515970395825, "max_ulp": 8579, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } } ], "comparison": { "actual_dtype": "float32", "actual_shape": [ 4, 83 ], "elements": 332, "expected_dtype": "float32", "expected_shape": [ 4, 83 ], "first_mismatch": null, "max_abs": 4.76837158203125e-07, "max_budget_ratio": 0.03118912304133686, "max_rel": 0.0005574756193776393, "max_ulp": 8578, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, "comparison_key": "105fe45d213d221e0c303861a2bb9c47643faaa68d8d52c9f96495396d708d30", "contract": "layer_norm", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "float32", "shape": [ 4, 83 ] } }, "ir_version": 10, "name": "LayerNormalization/random-003/float32/4x83/seed-20260906", "observation_id": "6801cd9914b32dd5ce8bbae833c8537607bf4e99e834282d36bd4719d46affd1", "operation": "LayerNormalization", "operators": [ "LayerNormalization" ], "opset": 19, "optimization": "all", "output_hash": "3f0e54857fa4ab65578470b4ba4d85e58e410dcbb69ea42206b14c6e25c28507", "outputs": {}, "parameters": { "epsilon": 1e-05 }, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.003208, "p95_ms": 0.0037464, "repeats": 7, "samples_ms": [ 0.003542, 0.003834, 0.003458, 0.003208, 0.003083, 0.003208, 0.003042 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.017041, "p95_ms": 0.018475, "repeats": 7, "samples_ms": [ 0.018625, 0.018125, 0.017417, 0.016917, 0.016917, 0.017041, 0.016583 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-06, "rtol": 1e-05 }, "variant": "random-003", "warnings": [] }, { "artifacts": { "bundle": "observations/8a8ec2daccb7f2a2631ab966d45ed0183d082a9305300868f7eceea90e719024/reproducer.zip", "bundle_sha256": "1e313d61407f2541b5ce3c7dfb82e1dc96f94e1e071dcd5acab9fba75f9e001f", "case": "artifacts/ea6ee1fa428648655807b50b6ee7ce84519017e6b852c34b6f2a367537033014/case.json", "inputs": "artifacts/ea6ee1fa428648655807b50b6ee7ce84519017e6b852c34b6f2a367537033014/inputs.npz", "manifest": "artifacts/ea6ee1fa428648655807b50b6ee7ce84519017e6b852c34b6f2a367537033014/manifest.json", "model": "artifacts/ea6ee1fa428648655807b50b6ee7ce84519017e6b852c34b6f2a367537033014/model.onnx", "outputs": "observations/8a8ec2daccb7f2a2631ab966d45ed0183d082a9305300868f7eceea90e719024/outputs.npz" }, "candidate_group": "2e7f18e5d825e764ea2890c1747ece69fa26764698875d70cd8577c904a17708", "case_id": "ea6ee1fa428648655807b50b6ee7ce84519017e6b852c34b6f2a367537033014", "category": "normalization", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 83 ], "backend": "reference", "elements": 332, "expected_dtype": "float32", "expected_shape": [ 4, 83 ], "first_mismatch": null, "max_abs": 2.384185791015625e-07, "max_budget_ratio": 0.01743071600870713, "max_rel": 0.0003116543515307663, "max_ulp": 4794, "mismatched": 0, "name": "centered_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 1.8407762047978338e-07, "max_budget_ratio": 0.016738852809237422, "max_rel": 1.8413235842348725e-07, "max_ulp": 1658023806, "mismatched": 0, "name": "normalized_variance", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 83 ], "backend": "reference", "elements": 332, "expected_dtype": "float32", "expected_shape": [ 4, 83 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 83 ], "backend": "reference", "elements": 332, "expected_dtype": "float32", "expected_shape": [ 4, 83 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 83 ], "backend": "reference", "elements": 332, "expected_dtype": "float32", "expected_shape": [ 4, 83 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 83 ], "backend": "reference", "elements": 664, "expected_dtype": "float32", "expected_shape": [ 8, 83 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 83 ], "backend": "reference", "elements": 332, "expected_dtype": "float32", "expected_shape": [ 4, 83 ], "first_mismatch": null, "max_abs": 2.384185791015625e-07, "max_budget_ratio": 0.010991641252526538, "max_rel": 1.1671343373956611e-07, "max_ulp": 1, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 83 ], "backend": "ort", "elements": 332, "expected_dtype": "float32", "expected_shape": [ 4, 83 ], "first_mismatch": null, "max_abs": 2.384185791015625e-07, "max_budget_ratio": 0.01787772548103806, "max_rel": 0.00024599500754952437, "max_ulp": 3784, "mismatched": 0, "name": "centered_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 1.558245660282509e-07, "max_budget_ratio": 0.014169887807201943, "max_rel": 1.5587318724346586e-07, "max_ulp": 1403542915, "mismatched": 0, "name": "normalized_variance", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 83 ], "backend": "ort", "elements": 332, "expected_dtype": "float32", "expected_shape": [ 4, 83 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 83 ], "backend": "ort", "elements": 332, "expected_dtype": "float32", "expected_shape": [ 4, 83 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 83 ], "backend": "ort", "elements": 332, "expected_dtype": "float32", "expected_shape": [ 4, 83 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 83 ], "backend": "ort", "elements": 664, "expected_dtype": "float32", "expected_shape": [ 8, 83 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 83 ], "backend": "ort", "elements": 332, "expected_dtype": "float32", "expected_shape": [ 4, 83 ], "first_mismatch": null, "max_abs": 2.384185791015625e-07, "max_budget_ratio": 0.031192768713552702, "max_rel": 0.0005578515970395825, "max_ulp": 8579, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } } ], "comparison": { "actual_dtype": "float32", "actual_shape": [ 4, 83 ], "elements": 332, "expected_dtype": "float32", "expected_shape": [ 4, 83 ], "first_mismatch": null, "max_abs": 4.76837158203125e-07, "max_budget_ratio": 0.03118912304133686, "max_rel": 0.0005574756193776393, "max_ulp": 8578, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, "comparison_key": "4ccdb68c3c277b24bb88ef3765cb5c869c8d83a9d585b7a993d33994f470bd2c", "contract": "layer_norm", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "float32", "shape": [ 4, 83 ] } }, "ir_version": 10, "name": "LayerNormalization/random-003/float32/4x83/seed-20260906", "observation_id": "8a8ec2daccb7f2a2631ab966d45ed0183d082a9305300868f7eceea90e719024", "operation": "LayerNormalization", "operators": [ "LayerNormalization" ], "opset": 19, "optimization": "disabled", "output_hash": "3f0e54857fa4ab65578470b4ba4d85e58e410dcbb69ea42206b14c6e25c28507", "outputs": {}, "parameters": { "epsilon": 1e-05 }, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.00325, "p95_ms": 0.0036211, "repeats": 7, "samples_ms": [ 0.003125, 0.003709, 0.003416, 0.003209, 0.00325, 0.003375, 0.002917 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.017417, "p95_ms": 0.0188588, "repeats": 7, "samples_ms": [ 0.018959, 0.018291, 0.018625, 0.017417, 0.017084, 0.016875, 0.01725 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-06, "rtol": 1e-05 }, "variant": "random-003", "warnings": [] }, { "artifacts": { "bundle": "observations/1d16b7488761fc134c91ed47653c275f63d7cdc8837290d46b8983ed6faab3c0/reproducer.zip", "bundle_sha256": "9c156daf6f89f75f8eb13d57588958886058f8919d68eb2b1c88117fea4c950d", "case": "artifacts/187b6ad9fd8304a020d806d3c2ab963b21ecb479f7afaa65aba0ed536711fb38/case.json", "inputs": "artifacts/187b6ad9fd8304a020d806d3c2ab963b21ecb479f7afaa65aba0ed536711fb38/inputs.npz", "manifest": "artifacts/187b6ad9fd8304a020d806d3c2ab963b21ecb479f7afaa65aba0ed536711fb38/manifest.json", "model": "artifacts/187b6ad9fd8304a020d806d3c2ab963b21ecb479f7afaa65aba0ed536711fb38/model.onnx", "outputs": "observations/1d16b7488761fc134c91ed47653c275f63d7cdc8837290d46b8983ed6faab3c0/outputs.npz" }, "candidate_group": "24f97077827703160975a8119537a54d273efd8453b0fc4ffd064cd56c02ae25", "case_id": "187b6ad9fd8304a020d806d3c2ab963b21ecb479f7afaa65aba0ed536711fb38", "category": "normalization", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 41 ], "backend": "reference", "elements": 164, "expected_dtype": "float32", "expected_shape": [ 4, 41 ], "first_mismatch": null, "max_abs": 2.384185791015625e-07, "max_budget_ratio": 0.014785077315013398, "max_rel": 1.7695879673680902e-07, "max_ulp": 2, "mismatched": 0, "name": "centered_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 8.746281932037192e-08, "max_budget_ratio": 0.008691575901699309, "max_rel": 9.6505996313805e-08, "max_ulp": 787795041, "mismatched": 0, "name": "normalized_variance", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 41 ], "backend": "reference", "elements": 164, "expected_dtype": "float32", "expected_shape": [ 4, 41 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 41 ], "backend": "reference", "elements": 164, "expected_dtype": "float32", "expected_shape": [ 4, 41 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 41 ], "backend": "reference", "elements": 164, "expected_dtype": "float32", "expected_shape": [ 4, 41 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 41 ], "backend": "reference", "elements": 328, "expected_dtype": "float32", "expected_shape": [ 8, 41 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 41 ], "backend": "reference", "elements": 164, "expected_dtype": "float32", "expected_shape": [ 4, 41 ], "first_mismatch": null, "max_abs": 2.384185791015625e-07, "max_budget_ratio": 0.010038962417819045, "max_rel": 1.1790582743656814e-07, "max_ulp": 1, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 41 ], "backend": "ort", "elements": 164, "expected_dtype": "float32", "expected_shape": [ 4, 41 ], "first_mismatch": null, "max_abs": 2.384185791015625e-07, "max_budget_ratio": 0.05471268026342032, "max_rel": 2.654439219583391e-06, "max_ulp": 37, "mismatched": 0, "name": "centered_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 1.7202822533146644e-07, "max_budget_ratio": 0.01719377369076173, "max_rel": 1.910307650158018e-07, "max_ulp": 1549492503, "mismatched": 0, "name": "normalized_variance", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 41 ], "backend": "ort", "elements": 164, "expected_dtype": "float32", "expected_shape": [ 4, 41 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 41 ], "backend": "ort", "elements": 164, "expected_dtype": "float32", "expected_shape": [ 4, 41 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 41 ], "backend": "ort", "elements": 164, "expected_dtype": "float32", "expected_shape": [ 4, 41 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 41 ], "backend": "ort", "elements": 328, "expected_dtype": "float32", "expected_shape": [ 8, 41 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 41 ], "backend": "ort", "elements": 164, "expected_dtype": "float32", "expected_shape": [ 4, 41 ], "first_mismatch": null, "max_abs": 4.76837158203125e-07, "max_budget_ratio": 0.06949992917069985, "max_rel": 3.371864175284709e-06, "max_ulp": 47, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } } ], "comparison": { "actual_dtype": "float32", "actual_shape": [ 4, 41 ], "elements": 164, "expected_dtype": "float32", "expected_shape": [ 4, 41 ], "first_mismatch": null, "max_abs": 2.384185791015625e-07, "max_budget_ratio": 0.05175523961765051, "max_rel": 2.510956378804888e-06, "max_ulp": 35, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, "comparison_key": "b9bbd63d23de2210571229fedb6c587ec9ca9cd20252f2752018b54324f5143e", "contract": "layer_norm", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "float32", "shape": [ 4, 41 ] } }, "ir_version": 10, "name": "LayerNormalization/random-010/float32/4x41/seed-20260906", "observation_id": "1d16b7488761fc134c91ed47653c275f63d7cdc8837290d46b8983ed6faab3c0", "operation": "LayerNormalization", "operators": [ "LayerNormalization" ], "opset": 19, "optimization": "all", "output_hash": "b0c701e8b8500871bd667714b45e15dc76cde4103bd388a15abc59e15ab2286a", "outputs": {}, "parameters": { "epsilon": 1e-05 }, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.003041, "p95_ms": 0.0036034999999999995, "repeats": 7, "samples_ms": [ 0.002959, 0.003791, 0.003125, 0.003166, 0.003041, 0.003041, 0.002666 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.016375, "p95_ms": 0.018174799999999998, "repeats": 7, "samples_ms": [ 0.0185, 0.017416, 0.016417, 0.01625, 0.016084, 0.016375, 0.015791 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-06, "rtol": 1e-05 }, "variant": "random-010", "warnings": [] }, { "artifacts": { "bundle": "observations/43ff7e3c0acef5ae9afdc9d28f5911798cc90e9f60945a88446b291d4258d111/reproducer.zip", "bundle_sha256": "d973ac77c1261c082636d779fb2f85a448005d7679f823488f2bc14eabed3078", "case": "artifacts/187b6ad9fd8304a020d806d3c2ab963b21ecb479f7afaa65aba0ed536711fb38/case.json", "inputs": "artifacts/187b6ad9fd8304a020d806d3c2ab963b21ecb479f7afaa65aba0ed536711fb38/inputs.npz", "manifest": "artifacts/187b6ad9fd8304a020d806d3c2ab963b21ecb479f7afaa65aba0ed536711fb38/manifest.json", "model": "artifacts/187b6ad9fd8304a020d806d3c2ab963b21ecb479f7afaa65aba0ed536711fb38/model.onnx", "outputs": "observations/43ff7e3c0acef5ae9afdc9d28f5911798cc90e9f60945a88446b291d4258d111/outputs.npz" }, "candidate_group": "24f97077827703160975a8119537a54d273efd8453b0fc4ffd064cd56c02ae25", "case_id": "187b6ad9fd8304a020d806d3c2ab963b21ecb479f7afaa65aba0ed536711fb38", "category": "normalization", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 41 ], "backend": "reference", "elements": 164, "expected_dtype": "float32", "expected_shape": [ 4, 41 ], "first_mismatch": null, "max_abs": 2.384185791015625e-07, "max_budget_ratio": 0.014785077315013398, "max_rel": 1.7695879673680902e-07, "max_ulp": 2, "mismatched": 0, "name": "centered_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 8.746281932037192e-08, "max_budget_ratio": 0.008691575901699309, "max_rel": 9.6505996313805e-08, "max_ulp": 787795041, "mismatched": 0, "name": "normalized_variance", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 41 ], "backend": "reference", "elements": 164, "expected_dtype": "float32", "expected_shape": [ 4, 41 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 41 ], "backend": "reference", "elements": 164, "expected_dtype": "float32", "expected_shape": [ 4, 41 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 41 ], "backend": "reference", "elements": 164, "expected_dtype": "float32", "expected_shape": [ 4, 41 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 41 ], "backend": "reference", "elements": 328, "expected_dtype": "float32", "expected_shape": [ 8, 41 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 41 ], "backend": "reference", "elements": 164, "expected_dtype": "float32", "expected_shape": [ 4, 41 ], "first_mismatch": null, "max_abs": 2.384185791015625e-07, "max_budget_ratio": 0.010038962417819045, "max_rel": 1.1790582743656814e-07, "max_ulp": 1, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 41 ], "backend": "ort", "elements": 164, "expected_dtype": "float32", "expected_shape": [ 4, 41 ], "first_mismatch": null, "max_abs": 2.384185791015625e-07, "max_budget_ratio": 0.05471268026342032, "max_rel": 2.654439219583391e-06, "max_ulp": 37, "mismatched": 0, "name": "centered_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 1.7202822533146644e-07, "max_budget_ratio": 0.01719377369076173, "max_rel": 1.910307650158018e-07, "max_ulp": 1549492503, "mismatched": 0, "name": "normalized_variance", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 41 ], "backend": "ort", "elements": 164, "expected_dtype": "float32", "expected_shape": [ 4, 41 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 41 ], "backend": "ort", "elements": 164, "expected_dtype": "float32", "expected_shape": [ 4, 41 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 41 ], "backend": "ort", "elements": 164, "expected_dtype": "float32", "expected_shape": [ 4, 41 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 41 ], "backend": "ort", "elements": 328, "expected_dtype": "float32", "expected_shape": [ 8, 41 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 41 ], "backend": "ort", "elements": 164, "expected_dtype": "float32", "expected_shape": [ 4, 41 ], "first_mismatch": null, "max_abs": 4.76837158203125e-07, "max_budget_ratio": 0.06949992917069985, "max_rel": 3.371864175284709e-06, "max_ulp": 47, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } } ], "comparison": { "actual_dtype": "float32", "actual_shape": [ 4, 41 ], "elements": 164, "expected_dtype": "float32", "expected_shape": [ 4, 41 ], "first_mismatch": null, "max_abs": 2.384185791015625e-07, "max_budget_ratio": 0.05175523961765051, "max_rel": 2.510956378804888e-06, "max_ulp": 35, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, "comparison_key": "97f45771f2015fd6e47d9e8afe50a413ab86fb69249ca8285dd7e9de5e9c600a", "contract": "layer_norm", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "float32", "shape": [ 4, 41 ] } }, "ir_version": 10, "name": "LayerNormalization/random-010/float32/4x41/seed-20260906", "observation_id": "43ff7e3c0acef5ae9afdc9d28f5911798cc90e9f60945a88446b291d4258d111", "operation": "LayerNormalization", "operators": [ "LayerNormalization" ], "opset": 19, "optimization": "disabled", "output_hash": "b0c701e8b8500871bd667714b45e15dc76cde4103bd388a15abc59e15ab2286a", "outputs": {}, "parameters": { "epsilon": 1e-05 }, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.002959, "p95_ms": 0.0035374, "repeats": 7, "samples_ms": [ 0.002958, 0.003625, 0.003333, 0.003083, 0.002959, 0.002708, 0.002875 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.016667, "p95_ms": 0.018262399999999998, "repeats": 7, "samples_ms": [ 0.0185, 0.017708, 0.017, 0.016667, 0.016458, 0.016458, 0.0165 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-06, "rtol": 1e-05 }, "variant": "random-010", "warnings": [] }, { "artifacts": { "bundle": "observations/6dfba5ef22d0737d67ec5a7f766344865bcebebf4e4fcfddb2e5836c90810350/reproducer.zip", "bundle_sha256": "2c745106c8740ce7f2a8be80a946be6d6a19da43e86753099c54554a70f13176", "case": "artifacts/2e9e93892f6a3657b99a48d4922771ea68eb2944b4fdc295fb3340399ab47103/case.json", "inputs": "artifacts/2e9e93892f6a3657b99a48d4922771ea68eb2944b4fdc295fb3340399ab47103/inputs.npz", "manifest": "artifacts/2e9e93892f6a3657b99a48d4922771ea68eb2944b4fdc295fb3340399ab47103/manifest.json", "model": "artifacts/2e9e93892f6a3657b99a48d4922771ea68eb2944b4fdc295fb3340399ab47103/model.onnx", "outputs": "observations/6dfba5ef22d0737d67ec5a7f766344865bcebebf4e4fcfddb2e5836c90810350/outputs.npz" }, "candidate_group": "c81a54851d5cfa1eb208a8819edc1c90481c76c9e6e8f77dfe5be0a836ff3b5e", "case_id": "2e9e93892f6a3657b99a48d4922771ea68eb2944b4fdc295fb3340399ab47103", "category": "normalization", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 58 ], "backend": "reference", "elements": 232, "expected_dtype": "float32", "expected_shape": [ 4, 58 ], "first_mismatch": null, "max_abs": 2.384185791015625e-07, "max_budget_ratio": 0.02127896631207681, "max_rel": 1.4583171353135219e-05, "max_ulp": 184, "mismatched": 0, "name": "centered_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 2.0570832914845738e-07, "max_budget_ratio": 0.018700757198681944, "max_rel": 2.0570832918920583e-07, "max_ulp": 1852855909, "mismatched": 0, "name": "normalized_variance", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 58 ], "backend": "reference", "elements": 232, "expected_dtype": "float32", "expected_shape": [ 4, 58 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 58 ], "backend": "reference", "elements": 232, "expected_dtype": "float32", "expected_shape": [ 4, 58 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 58 ], "backend": "reference", "elements": 232, "expected_dtype": "float32", "expected_shape": [ 4, 58 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 58 ], "backend": "reference", "elements": 464, "expected_dtype": "float32", "expected_shape": [ 8, 58 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 58 ], "backend": "reference", "elements": 232, "expected_dtype": "float32", "expected_shape": [ 4, 58 ], "first_mismatch": null, "max_abs": 2.384185791015625e-07, "max_budget_ratio": 0.01071635121443693, "max_rel": 1.1682120164624437e-07, "max_ulp": 1, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 58 ], "backend": "ort", "elements": 232, "expected_dtype": "float32", "expected_shape": [ 4, 58 ], "first_mismatch": null, "max_abs": 2.384185791015625e-07, "max_budget_ratio": 0.021222382941686282, "max_rel": 3.607072748244758e-06, "max_ulp": 54, "mismatched": 0, "name": "centered_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 3.721206243145758e-08, "max_budget_ratio": 0.0033829147673467524, "max_rel": 3.721206244174995e-08, "max_ulp": 287431536, "mismatched": 0, "name": "normalized_variance", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 58 ], "backend": "ort", "elements": 232, "expected_dtype": "float32", "expected_shape": [ 4, 58 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 58 ], "backend": "ort", "elements": 232, "expected_dtype": "float32", "expected_shape": [ 4, 58 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 58 ], "backend": "ort", "elements": 232, "expected_dtype": "float32", "expected_shape": [ 4, 58 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 58 ], "backend": "ort", "elements": 464, "expected_dtype": "float32", "expected_shape": [ 8, 58 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 58 ], "backend": "ort", "elements": 232, "expected_dtype": "float32", "expected_shape": [ 4, 58 ], "first_mismatch": null, "max_abs": 2.384185791015625e-07, "max_budget_ratio": 0.034486364961411775, "max_rel": 5.944206724609737e-06, "max_ulp": 75, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } } ], "comparison": { "actual_dtype": "float32", "actual_shape": [ 4, 58 ], "elements": 232, "expected_dtype": "float32", "expected_shape": [ 4, 58 ], "first_mismatch": null, "max_abs": 3.5762786865234375e-07, "max_budget_ratio": 0.028388195232514363, "max_rel": 1.8070714987652866e-05, "max_ulp": 228, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, "comparison_key": "687f518349de4d462a426b5d94d564d47f2c7d721022ff1889a8f8479d492a95", "contract": "layer_norm", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "float32", "shape": [ 4, 58 ] } }, "ir_version": 10, "name": "LayerNormalization/random-017/float32/4x58/seed-20260906", "observation_id": "6dfba5ef22d0737d67ec5a7f766344865bcebebf4e4fcfddb2e5836c90810350", "operation": "LayerNormalization", "operators": [ "LayerNormalization" ], "opset": 19, "optimization": "all", "output_hash": "9df94e0c961f83fb35133bdca7f2810fb338f52630b9b9fbb17e107b2495e245", "outputs": {}, "parameters": { "epsilon": 1e-05 }, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.003209, "p95_ms": 0.0034376, "repeats": 7, "samples_ms": [ 0.003292, 0.0035, 0.003208, 0.003167, 0.003209, 0.003291, 0.003125 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.016833, "p95_ms": 0.0182669, "repeats": 7, "samples_ms": [ 0.018542, 0.017458, 0.016833, 0.016417, 0.016292, 0.017625, 0.01625 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-06, "rtol": 1e-05 }, "variant": "random-017", "warnings": [] }, { "artifacts": { "bundle": "observations/5b046810f4160e7282cd97854475d77828b25ee532dd7d4580c3ca1130408c36/reproducer.zip", "bundle_sha256": "e53a1cc64e790b4cb49064f5d5155a92eefb1cab68c971b218744c466d576e4d", "case": "artifacts/2e9e93892f6a3657b99a48d4922771ea68eb2944b4fdc295fb3340399ab47103/case.json", "inputs": "artifacts/2e9e93892f6a3657b99a48d4922771ea68eb2944b4fdc295fb3340399ab47103/inputs.npz", "manifest": "artifacts/2e9e93892f6a3657b99a48d4922771ea68eb2944b4fdc295fb3340399ab47103/manifest.json", "model": "artifacts/2e9e93892f6a3657b99a48d4922771ea68eb2944b4fdc295fb3340399ab47103/model.onnx", "outputs": "observations/5b046810f4160e7282cd97854475d77828b25ee532dd7d4580c3ca1130408c36/outputs.npz" }, "candidate_group": "c81a54851d5cfa1eb208a8819edc1c90481c76c9e6e8f77dfe5be0a836ff3b5e", "case_id": "2e9e93892f6a3657b99a48d4922771ea68eb2944b4fdc295fb3340399ab47103", "category": "normalization", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 58 ], "backend": "reference", "elements": 232, "expected_dtype": "float32", "expected_shape": [ 4, 58 ], "first_mismatch": null, "max_abs": 2.384185791015625e-07, "max_budget_ratio": 0.02127896631207681, "max_rel": 1.4583171353135219e-05, "max_ulp": 184, "mismatched": 0, "name": "centered_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 2.0570832914845738e-07, "max_budget_ratio": 0.018700757198681944, "max_rel": 2.0570832918920583e-07, "max_ulp": 1852855909, "mismatched": 0, "name": "normalized_variance", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 58 ], "backend": "reference", "elements": 232, "expected_dtype": "float32", "expected_shape": [ 4, 58 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 58 ], "backend": "reference", "elements": 232, "expected_dtype": "float32", "expected_shape": [ 4, 58 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 58 ], "backend": "reference", "elements": 232, "expected_dtype": "float32", "expected_shape": [ 4, 58 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 58 ], "backend": "reference", "elements": 464, "expected_dtype": "float32", "expected_shape": [ 8, 58 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 58 ], "backend": "reference", "elements": 232, "expected_dtype": "float32", "expected_shape": [ 4, 58 ], "first_mismatch": null, "max_abs": 2.384185791015625e-07, "max_budget_ratio": 0.01071635121443693, "max_rel": 1.1682120164624437e-07, "max_ulp": 1, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 58 ], "backend": "ort", "elements": 232, "expected_dtype": "float32", "expected_shape": [ 4, 58 ], "first_mismatch": null, "max_abs": 2.384185791015625e-07, "max_budget_ratio": 0.021222382941686282, "max_rel": 3.607072748244758e-06, "max_ulp": 54, "mismatched": 0, "name": "centered_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 3.721206243145758e-08, "max_budget_ratio": 0.0033829147673467524, "max_rel": 3.721206244174995e-08, "max_ulp": 287431536, "mismatched": 0, "name": "normalized_variance", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 58 ], "backend": "ort", "elements": 232, "expected_dtype": "float32", "expected_shape": [ 4, 58 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 58 ], "backend": "ort", "elements": 232, "expected_dtype": "float32", "expected_shape": [ 4, 58 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 58 ], "backend": "ort", "elements": 232, "expected_dtype": "float32", "expected_shape": [ 4, 58 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 58 ], "backend": "ort", "elements": 464, "expected_dtype": "float32", "expected_shape": [ 8, 58 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 58 ], "backend": "ort", "elements": 232, "expected_dtype": "float32", "expected_shape": [ 4, 58 ], "first_mismatch": null, "max_abs": 2.384185791015625e-07, "max_budget_ratio": 0.034486364961411775, "max_rel": 5.944206724609737e-06, "max_ulp": 75, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } } ], "comparison": { "actual_dtype": "float32", "actual_shape": [ 4, 58 ], "elements": 232, "expected_dtype": "float32", "expected_shape": [ 4, 58 ], "first_mismatch": null, "max_abs": 3.5762786865234375e-07, "max_budget_ratio": 0.028388195232514363, "max_rel": 1.8070714987652866e-05, "max_ulp": 228, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, "comparison_key": "cf856f6957bec4cbfd29944097af2cdbd120f4768795e2f93d307f03974a59af", "contract": "layer_norm", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "float32", "shape": [ 4, 58 ] } }, "ir_version": 10, "name": "LayerNormalization/random-017/float32/4x58/seed-20260906", "observation_id": "5b046810f4160e7282cd97854475d77828b25ee532dd7d4580c3ca1130408c36", "operation": "LayerNormalization", "operators": [ "LayerNormalization" ], "opset": 19, "optimization": "disabled", "output_hash": "9df94e0c961f83fb35133bdca7f2810fb338f52630b9b9fbb17e107b2495e245", "outputs": {}, "parameters": { "epsilon": 1e-05 }, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.00325, "p95_ms": 0.0037545, "repeats": 7, "samples_ms": [ 0.003667, 0.003792, 0.003625, 0.00325, 0.003083, 0.003, 0.002833 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.016875, "p95_ms": 0.0188874, "repeats": 7, "samples_ms": [ 0.019125, 0.018333, 0.017417, 0.016875, 0.016541, 0.016459, 0.016541 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-06, "rtol": 1e-05 }, "variant": "random-017", "warnings": [] }, { "artifacts": { "bundle": "observations/29fe0e107d048edacdb54efd29ad0d106428f73bd56761bbf2a426fb291ba141/reproducer.zip", "bundle_sha256": "d2f70a9c491c5cbae44fc61168f6beaf3e3fcf32da70b163a0f695e44a80887e", "case": "artifacts/1a329aa1cafa58cf5693a52285776ffe1562183af23b75667e7c4803bf4b489f/case.json", "inputs": "artifacts/1a329aa1cafa58cf5693a52285776ffe1562183af23b75667e7c4803bf4b489f/inputs.npz", "manifest": "artifacts/1a329aa1cafa58cf5693a52285776ffe1562183af23b75667e7c4803bf4b489f/manifest.json", "model": "artifacts/1a329aa1cafa58cf5693a52285776ffe1562183af23b75667e7c4803bf4b489f/model.onnx", "outputs": "observations/29fe0e107d048edacdb54efd29ad0d106428f73bd56761bbf2a426fb291ba141/outputs.npz" }, "candidate_group": "45a4149aa6541f53deb3ef6463db96d87033eac99207ac3528f16260d503e924", "case_id": "1a329aa1cafa58cf5693a52285776ffe1562183af23b75667e7c4803bf4b489f", "category": "normalization", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 3.725290298461914e-09, "max_budget_ratio": 0.002726143142426727, "max_rel": 4.3869943166488627e-07, "max_ulp": 5, "mismatched": 0, "name": "centered_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 3.995615568009453e-11, "max_budget_ratio": 3.958776183905515e-05, "max_rel": 4.293705808449481e-08, "max_ulp": 368530489, "mismatched": 0, "name": "normalized_variance", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 17 ], "backend": "reference", "elements": 136, "expected_dtype": "float32", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 3.725290298461914e-09, "max_budget_ratio": 0.0027849750351774376, "max_rel": 1.1033363899429122e-07, "max_ulp": 1, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 3.725290298461914e-09, "max_budget_ratio": 0.0027849750351774376, "max_rel": 4.005542388892663e-07, "max_ulp": 5, "mismatched": 0, "name": "centered_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 7.415022003527671e-11, "max_budget_ratio": 7.351548482697859e-05, "max_rel": 8.58813140448982e-08, "max_ulp": 683915066, "mismatched": 0, "name": "normalized_variance", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 17 ], "backend": "ort", "elements": 136, "expected_dtype": "float32", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 7.450580596923828e-09, "max_budget_ratio": 0.0052855677841218505, "max_rel": 1.8644723800900234e-06, "max_ulp": 17, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } } ], "comparison": { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 7.450580596923828e-09, "max_budget_ratio": 0.005285568063494134, "max_rel": 8.011087986660577e-07, "max_ulp": 10, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, "comparison_key": "62a7915ad569cfca30fe37ea209d1a690ea219d9b6cc6d3b9263a76f1f143c52", "contract": "layer_norm", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "float32", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "LayerNormalization/tiny/float32/4x17/seed-20260906", "observation_id": "29fe0e107d048edacdb54efd29ad0d106428f73bd56761bbf2a426fb291ba141", "operation": "LayerNormalization", "operators": [ "LayerNormalization" ], "opset": 19, "optimization": "all", "output_hash": "17920c8859eb8c10245318e4c907cc131231466f194d21409bec21a353b62c03", "outputs": {}, "parameters": { "epsilon": 1e-05 }, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.003416, "p95_ms": 0.0037787, "repeats": 7, "samples_ms": [ 0.003791, 0.00375, 0.003416, 0.003541, 0.002958, 0.003125, 0.002792 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.016833, "p95_ms": 0.0174669, "repeats": 7, "samples_ms": [ 0.017667, 0.017, 0.016833, 0.016375, 0.016125, 0.016042, 0.016875 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-06, "rtol": 1e-05 }, "variant": "tiny", "warnings": [] }, { "artifacts": { "bundle": "observations/4fad27efad4e549167f0b955ca6b2bd673e33da8e21fa5fd4d56a22311f49a2e/reproducer.zip", "bundle_sha256": "506700e2368e8a942e00eae830caf62d67a6b7a1acf81a889440406aff12f7a2", "case": "artifacts/1a329aa1cafa58cf5693a52285776ffe1562183af23b75667e7c4803bf4b489f/case.json", "inputs": "artifacts/1a329aa1cafa58cf5693a52285776ffe1562183af23b75667e7c4803bf4b489f/inputs.npz", "manifest": "artifacts/1a329aa1cafa58cf5693a52285776ffe1562183af23b75667e7c4803bf4b489f/manifest.json", "model": "artifacts/1a329aa1cafa58cf5693a52285776ffe1562183af23b75667e7c4803bf4b489f/model.onnx", "outputs": "observations/4fad27efad4e549167f0b955ca6b2bd673e33da8e21fa5fd4d56a22311f49a2e/outputs.npz" }, "candidate_group": "45a4149aa6541f53deb3ef6463db96d87033eac99207ac3528f16260d503e924", "case_id": "1a329aa1cafa58cf5693a52285776ffe1562183af23b75667e7c4803bf4b489f", "category": "normalization", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 3.725290298461914e-09, "max_budget_ratio": 0.002726143142426727, "max_rel": 4.3869943166488627e-07, "max_ulp": 5, "mismatched": 0, "name": "centered_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 3.995615568009453e-11, "max_budget_ratio": 3.958776183905515e-05, "max_rel": 4.293705808449481e-08, "max_ulp": 368530489, "mismatched": 0, "name": "normalized_variance", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 17 ], "backend": "reference", "elements": 136, "expected_dtype": "float32", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 3.725290298461914e-09, "max_budget_ratio": 0.0027849750351774376, "max_rel": 1.1033363899429122e-07, "max_ulp": 1, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 3.725290298461914e-09, "max_budget_ratio": 0.0027849750351774376, "max_rel": 4.005542388892663e-07, "max_ulp": 5, "mismatched": 0, "name": "centered_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 7.415022003527671e-11, "max_budget_ratio": 7.351548482697859e-05, "max_rel": 8.58813140448982e-08, "max_ulp": 683915066, "mismatched": 0, "name": "normalized_variance", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 17 ], "backend": "ort", "elements": 136, "expected_dtype": "float32", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 7.450580596923828e-09, "max_budget_ratio": 0.0052855677841218505, "max_rel": 1.8644723800900234e-06, "max_ulp": 17, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } } ], "comparison": { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 7.450580596923828e-09, "max_budget_ratio": 0.005285568063494134, "max_rel": 8.011087986660577e-07, "max_ulp": 10, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, "comparison_key": "e9b13d507990d55f4a900c08e2c2b1a883090c1653c1ddb1f144cd1c906550a5", "contract": "layer_norm", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "float32", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "LayerNormalization/tiny/float32/4x17/seed-20260906", "observation_id": "4fad27efad4e549167f0b955ca6b2bd673e33da8e21fa5fd4d56a22311f49a2e", "operation": "LayerNormalization", "operators": [ "LayerNormalization" ], "opset": 19, "optimization": "disabled", "output_hash": "17920c8859eb8c10245318e4c907cc131231466f194d21409bec21a353b62c03", "outputs": {}, "parameters": { "epsilon": 1e-05 }, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.003, "p95_ms": 0.0032419, "repeats": 7, "samples_ms": [ 0.003125, 0.003292, 0.003, 0.002875, 0.003041, 0.002667, 0.00275 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.016333, "p95_ms": 0.017562599999999998, "repeats": 7, "samples_ms": [ 0.017625, 0.017417, 0.016667, 0.016333, 0.016333, 0.016292, 0.016083 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-06, "rtol": 1e-05 }, "variant": "tiny", "warnings": [] }, { "artifacts": { "bundle": "observations/f75d33a4e8676924236867860872441e152a7fa2ffa0829cde7b5893d0cb8163/reproducer.zip", "bundle_sha256": "c0949b5102898173a41a50a9c7e5a171313ae09c5ca04637454c841a61062813", "case": "artifacts/4ed1d5ed9652202c1cd16647ca3c149b0a77fa4ec137392df200cf9e5bfc3c55/case.json", "inputs": "artifacts/4ed1d5ed9652202c1cd16647ca3c149b0a77fa4ec137392df200cf9e5bfc3c55/inputs.npz", "manifest": "artifacts/4ed1d5ed9652202c1cd16647ca3c149b0a77fa4ec137392df200cf9e5bfc3c55/manifest.json", "model": "artifacts/4ed1d5ed9652202c1cd16647ca3c149b0a77fa4ec137392df200cf9e5bfc3c55/model.onnx", "outputs": "observations/f75d33a4e8676924236867860872441e152a7fa2ffa0829cde7b5893d0cb8163/outputs.npz" }, "candidate_group": "b6635553ab8d3530ee7771f64b8bd850441d6813af0ba57b75d87327cbea6436", "case_id": "4ed1d5ed9652202c1cd16647ca3c149b0a77fa4ec137392df200cf9e5bfc3c55", "category": "normalization", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 257 ], "backend": "reference", "elements": 1028, "expected_dtype": "float32", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 4.76837158203125e-07, "max_budget_ratio": 0.014542833873907432, "max_rel": 5.83625624394306e-06, "max_ulp": 60, "mismatched": 0, "name": "centered_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 7.991581496558098e-08, "max_budget_ratio": 0.007265137520153898, "max_rel": 7.991658249797422e-08, "max_ulp": 719817669, "mismatched": 0, "name": "normalized_variance", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 257 ], "backend": "reference", "elements": 1028, "expected_dtype": "float32", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 257 ], "backend": "reference", "elements": 1028, "expected_dtype": "float32", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 257 ], "backend": "reference", "elements": 1028, "expected_dtype": "float32", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 257 ], "backend": "reference", "elements": 2056, "expected_dtype": "float32", "expected_shape": [ 8, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 257 ], "backend": "reference", "elements": 1028, "expected_dtype": "float32", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 4.76837158203125e-07, "max_budget_ratio": 0.011516429015173035, "max_rel": 1.187989050542519e-07, "max_ulp": 1, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 257 ], "backend": "ort", "elements": 1028, "expected_dtype": "float32", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 4.76837158203125e-07, "max_budget_ratio": 0.018606806436779075, "max_rel": 5.490123693433386e-06, "max_ulp": 58, "mismatched": 0, "name": "centered_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 1.682503149025294e-07, "max_budget_ratio": 0.015295645192547504, "max_rel": 1.682522753416485e-07, "max_ulp": 1515464111, "mismatched": 0, "name": "normalized_variance", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 257 ], "backend": "ort", "elements": 1028, "expected_dtype": "float32", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 257 ], "backend": "ort", "elements": 1028, "expected_dtype": "float32", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 257 ], "backend": "ort", "elements": 1028, "expected_dtype": "float32", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 257 ], "backend": "ort", "elements": 2056, "expected_dtype": "float32", "expected_shape": [ 8, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 257 ], "backend": "ort", "elements": 1028, "expected_dtype": "float32", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 4.76837158203125e-07, "max_budget_ratio": 0.021725060152346546, "max_rel": 6.436661405958058e-06, "max_ulp": 68, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } } ], "comparison": { "actual_dtype": "float32", "actual_shape": [ 4, 257 ], "elements": 1028, "expected_dtype": "float32", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 2.384185791015625e-07, "max_budget_ratio": 0.016991264671893325, "max_rel": 3.112651830535951e-06, "max_ulp": 32, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, "comparison_key": "d47ee9885a2fe43582ed6526244d3b9ad02fcb2b51bb8b521f80492f6b2fdbbe", "contract": "layer_norm", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "float32", "shape": [ 4, 257 ] } }, "ir_version": 10, "name": "LayerNormalization/wide/float32/4x257/seed-20260906", "observation_id": "f75d33a4e8676924236867860872441e152a7fa2ffa0829cde7b5893d0cb8163", "operation": "LayerNormalization", "operators": [ "LayerNormalization" ], "opset": 19, "optimization": "all", "output_hash": "4e36d103d2d6fdec7b62deeca6c50113935bcc03bd261ea49e964b96f8b49eae", "outputs": {}, "parameters": { "epsilon": 1e-05 }, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.003958, "p95_ms": 0.0040294, "repeats": 7, "samples_ms": [ 0.003959, 0.004, 0.003958, 0.004042, 0.003709, 0.003791, 0.003583 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.017792, "p95_ms": 0.0189456, "repeats": 7, "samples_ms": [ 0.019083, 0.018625, 0.018083, 0.01775, 0.017792, 0.01775, 0.017166 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-06, "rtol": 1e-05 }, "variant": "wide", "warnings": [] }, { "artifacts": { "bundle": "observations/5b9181aad2dfe6bf366b3fb76e3341a625170b0b385de5f5d7cb7eb6f6e72a38/reproducer.zip", "bundle_sha256": "0218b057b55e267ce335b322c786ed09e72b997cce563d1e83133c0233053778", "case": "artifacts/4ed1d5ed9652202c1cd16647ca3c149b0a77fa4ec137392df200cf9e5bfc3c55/case.json", "inputs": "artifacts/4ed1d5ed9652202c1cd16647ca3c149b0a77fa4ec137392df200cf9e5bfc3c55/inputs.npz", "manifest": "artifacts/4ed1d5ed9652202c1cd16647ca3c149b0a77fa4ec137392df200cf9e5bfc3c55/manifest.json", "model": "artifacts/4ed1d5ed9652202c1cd16647ca3c149b0a77fa4ec137392df200cf9e5bfc3c55/model.onnx", "outputs": "observations/5b9181aad2dfe6bf366b3fb76e3341a625170b0b385de5f5d7cb7eb6f6e72a38/outputs.npz" }, "candidate_group": "b6635553ab8d3530ee7771f64b8bd850441d6813af0ba57b75d87327cbea6436", "case_id": "4ed1d5ed9652202c1cd16647ca3c149b0a77fa4ec137392df200cf9e5bfc3c55", "category": "normalization", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 257 ], "backend": "reference", "elements": 1028, "expected_dtype": "float32", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 4.76837158203125e-07, "max_budget_ratio": 0.014542833873907432, "max_rel": 5.83625624394306e-06, "max_ulp": 60, "mismatched": 0, "name": "centered_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 7.991581496558098e-08, "max_budget_ratio": 0.007265137520153898, "max_rel": 7.991658249797422e-08, "max_ulp": 719817669, "mismatched": 0, "name": "normalized_variance", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 257 ], "backend": "reference", "elements": 1028, "expected_dtype": "float32", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 257 ], "backend": "reference", "elements": 1028, "expected_dtype": "float32", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 257 ], "backend": "reference", "elements": 1028, "expected_dtype": "float32", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 257 ], "backend": "reference", "elements": 2056, "expected_dtype": "float32", "expected_shape": [ 8, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 257 ], "backend": "reference", "elements": 1028, "expected_dtype": "float32", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 4.76837158203125e-07, "max_budget_ratio": 0.011516429015173035, "max_rel": 1.187989050542519e-07, "max_ulp": 1, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 257 ], "backend": "ort", "elements": 1028, "expected_dtype": "float32", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 4.76837158203125e-07, "max_budget_ratio": 0.018606806436779075, "max_rel": 5.490123693433386e-06, "max_ulp": 58, "mismatched": 0, "name": "centered_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 1.682503149025294e-07, "max_budget_ratio": 0.015295645192547504, "max_rel": 1.682522753416485e-07, "max_ulp": 1515464111, "mismatched": 0, "name": "normalized_variance", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 257 ], "backend": "ort", "elements": 1028, "expected_dtype": "float32", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 257 ], "backend": "ort", "elements": 1028, "expected_dtype": "float32", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 257 ], "backend": "ort", "elements": 1028, "expected_dtype": "float32", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 257 ], "backend": "ort", "elements": 2056, "expected_dtype": "float32", "expected_shape": [ 8, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 257 ], "backend": "ort", "elements": 1028, "expected_dtype": "float32", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 4.76837158203125e-07, "max_budget_ratio": 0.021725060152346546, "max_rel": 6.436661405958058e-06, "max_ulp": 68, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } } ], "comparison": { "actual_dtype": "float32", "actual_shape": [ 4, 257 ], "elements": 1028, "expected_dtype": "float32", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 2.384185791015625e-07, "max_budget_ratio": 0.016991264671893325, "max_rel": 3.112651830535951e-06, "max_ulp": 32, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, "comparison_key": "bd13fd96d8d55b6f1b7fe0c52eb931463953d23064e647a68ecac8ed6f41d689", "contract": "layer_norm", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "float32", "shape": [ 4, 257 ] } }, "ir_version": 10, "name": "LayerNormalization/wide/float32/4x257/seed-20260906", "observation_id": "5b9181aad2dfe6bf366b3fb76e3341a625170b0b385de5f5d7cb7eb6f6e72a38", "operation": "LayerNormalization", "operators": [ "LayerNormalization" ], "opset": 19, "optimization": "disabled", "output_hash": "4e36d103d2d6fdec7b62deeca6c50113935bcc03bd261ea49e964b96f8b49eae", "outputs": {}, "parameters": { "epsilon": 1e-05 }, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.004167, "p95_ms": 0.0045125, "repeats": 7, "samples_ms": [ 0.004167, 0.004625, 0.00425, 0.004042, 0.004208, 0.003792, 0.003542 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.017875, "p95_ms": 0.0185713, "repeats": 7, "samples_ms": [ 0.018709, 0.01825, 0.017875, 0.018083, 0.017667, 0.01775, 0.017291 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-06, "rtol": 1e-05 }, "variant": "wide", "warnings": [] }, { "artifacts": { "bundle": "observations/ba9fb2d486406957d5ba8fb86555e3ea6db549376e6115ede77b5eb623d238d5/reproducer.zip", "bundle_sha256": "03f24dda74c4e7ab43058ea2c367db06ddf3d9c7ce7489f469265dd3bc17637e", "case": "artifacts/415667bfd1ef2bd346c17a25aafeef0d671d90b22d03bb0150a7ac8f56002d16/case.json", "inputs": "artifacts/415667bfd1ef2bd346c17a25aafeef0d671d90b22d03bb0150a7ac8f56002d16/inputs.npz", "manifest": "artifacts/415667bfd1ef2bd346c17a25aafeef0d671d90b22d03bb0150a7ac8f56002d16/manifest.json", "model": "artifacts/415667bfd1ef2bd346c17a25aafeef0d671d90b22d03bb0150a7ac8f56002d16/model.onnx", "outputs": "observations/ba9fb2d486406957d5ba8fb86555e3ea6db549376e6115ede77b5eb623d238d5/outputs.npz" }, "candidate_group": "9326a75026603e9e54c12bc20058987704f0bd1433c71ee384325a373d0aa79a", "case_id": "415667bfd1ef2bd346c17a25aafeef0d671d90b22d03bb0150a7ac8f56002d16", "category": "softmax", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "backend": "reference", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 0.0011827930047754176, "max_budget_ratio": 0.21505327359553048, "max_rel": 0.0011827930047754176, "max_ulp": 5326826135563, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 8, 17 ], "backend": "reference", "elements": 136, "expected_dtype": "float16", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.001953125, "max_budget_ratio": 0.1332267519317879, "max_rel": 0.000689655172413793, "max_ulp": 1, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "backend": "ort", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 0.0007707337758059429, "max_budget_ratio": 0.14013341378289873, "max_rel": 0.0007707337758059429, "max_ulp": 6942152691043, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 8, 17 ], "backend": "ort", "elements": 136, "expected_dtype": "float16", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } } ], "comparison": { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.001953125, "max_budget_ratio": 0.1332267519317879, "max_rel": 0.000689655172413793, "max_ulp": 1, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, "comparison_key": "cd715c0014b097c5091d22b185c9cb30589d3c18c5cc39418b9c6890d759156a", "contract": "log_probability", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "float16", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "LogSoftmax/constant/float16/4x17/seed-20260906", "observation_id": "ba9fb2d486406957d5ba8fb86555e3ea6db549376e6115ede77b5eb623d238d5", "operation": "LogSoftmax", "operators": [ "LogSoftmax" ], "opset": 19, "optimization": "all", "output_hash": "8f8b323deb17321f039b176e26faf1e4f938c871610e3f35f1f06588a9407d16", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.004542, "p95_ms": 0.0056749999999999995, "repeats": 7, "samples_ms": [ 0.00575, 0.0055, 0.005, 0.004542, 0.004333, 0.004292, 0.004333 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.009791, "p95_ms": 0.010700099999999999, "repeats": 7, "samples_ms": [ 0.010875, 0.010292, 0.01, 0.009709, 0.009417, 0.009791, 0.009417 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 0.0005, "rtol": 0.005 }, "variant": "constant", "warnings": [] }, { "artifacts": { "bundle": "observations/e82e3eb4886d3d1a2db51244a7fb16d31869cf65c4bb71f3f684eec5a7e57970/reproducer.zip", "bundle_sha256": "5422e6e4a38d13364b594dbaa54f0196ebac49b7cb27fe5371d4135f56e791aa", "case": "artifacts/415667bfd1ef2bd346c17a25aafeef0d671d90b22d03bb0150a7ac8f56002d16/case.json", "inputs": "artifacts/415667bfd1ef2bd346c17a25aafeef0d671d90b22d03bb0150a7ac8f56002d16/inputs.npz", "manifest": "artifacts/415667bfd1ef2bd346c17a25aafeef0d671d90b22d03bb0150a7ac8f56002d16/manifest.json", "model": "artifacts/415667bfd1ef2bd346c17a25aafeef0d671d90b22d03bb0150a7ac8f56002d16/model.onnx", "outputs": "observations/e82e3eb4886d3d1a2db51244a7fb16d31869cf65c4bb71f3f684eec5a7e57970/outputs.npz" }, "candidate_group": "9326a75026603e9e54c12bc20058987704f0bd1433c71ee384325a373d0aa79a", "case_id": "415667bfd1ef2bd346c17a25aafeef0d671d90b22d03bb0150a7ac8f56002d16", "category": "softmax", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "backend": "reference", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 0.0011827930047754176, "max_budget_ratio": 0.21505327359553048, "max_rel": 0.0011827930047754176, "max_ulp": 5326826135563, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 8, 17 ], "backend": "reference", "elements": 136, "expected_dtype": "float16", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.001953125, "max_budget_ratio": 0.1332267519317879, "max_rel": 0.000689655172413793, "max_ulp": 1, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "backend": "ort", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 0.0007707337758059429, "max_budget_ratio": 0.14013341378289873, "max_rel": 0.0007707337758059429, "max_ulp": 6942152691043, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 8, 17 ], "backend": "ort", "elements": 136, "expected_dtype": "float16", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } } ], "comparison": { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.001953125, "max_budget_ratio": 0.1332267519317879, "max_rel": 0.000689655172413793, "max_ulp": 1, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, "comparison_key": "331fa6f1b2be7afeb8b8282b5217b9068cd4a458aa558f01e4f45f84fdf381be", "contract": "log_probability", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "float16", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "LogSoftmax/constant/float16/4x17/seed-20260906", "observation_id": "e82e3eb4886d3d1a2db51244a7fb16d31869cf65c4bb71f3f684eec5a7e57970", "operation": "LogSoftmax", "operators": [ "LogSoftmax" ], "opset": 19, "optimization": "disabled", "output_hash": "8f8b323deb17321f039b176e26faf1e4f938c871610e3f35f1f06588a9407d16", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.005375, "p95_ms": 0.0061043, "repeats": 7, "samples_ms": [ 0.005542, 0.005375, 0.005375, 0.006167, 0.005958, 0.005333, 0.004584 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.009625, "p95_ms": 0.0104624, "repeats": 7, "samples_ms": [ 0.010625, 0.010083, 0.009792, 0.009542, 0.0095, 0.009625, 0.00925 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 0.0005, "rtol": 0.005 }, "variant": "constant", "warnings": [] }, { "artifacts": { "bundle": "observations/d23cd7a0ed75f5fed8a55019d8b31869eda413ae9a1db9d80374073b3cc9fb6a/reproducer.zip", "bundle_sha256": "3681020ac78e18e83ae8e74c41d40a8a0525d4e4df74e0ccddf0ac85cee0fb89", "case": "artifacts/12990c39730af0d8399d7b2b94c3e0b186a85075838e7dfcdad80ac7a08e8ddd/case.json", "inputs": "artifacts/12990c39730af0d8399d7b2b94c3e0b186a85075838e7dfcdad80ac7a08e8ddd/inputs.npz", "manifest": "artifacts/12990c39730af0d8399d7b2b94c3e0b186a85075838e7dfcdad80ac7a08e8ddd/manifest.json", "model": "artifacts/12990c39730af0d8399d7b2b94c3e0b186a85075838e7dfcdad80ac7a08e8ddd/model.onnx", "outputs": "observations/d23cd7a0ed75f5fed8a55019d8b31869eda413ae9a1db9d80374073b3cc9fb6a/outputs.npz" }, "candidate_group": "39719b3019cdeb423ee47d31af176e009a2dd30eac0a5329f188c48390331561", "case_id": "12990c39730af0d8399d7b2b94c3e0b186a85075838e7dfcdad80ac7a08e8ddd", "category": "softmax", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "backend": "reference", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 1.4741030618381501e-08, "max_budget_ratio": 0.0013400936925801364, "max_rel": 1.4741030618381501e-08, "max_ulp": 66387700, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 17 ], "backend": "reference", "elements": 136, "expected_dtype": "float32", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "backend": "ort", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 1.4741030618381501e-08, "max_budget_ratio": 0.0013400936925801364, "max_rel": 1.4741030618381501e-08, "max_ulp": 66387700, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 17 ], "backend": "ort", "elements": 136, "expected_dtype": "float32", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } } ], "comparison": { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, "comparison_key": "6dc4672f779fd9090b1f02a22640b3bab27b7fcf1be0653983c66f1edc57b148", "contract": "log_probability", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "float32", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "LogSoftmax/constant/float32/4x17/seed-20260906", "observation_id": "d23cd7a0ed75f5fed8a55019d8b31869eda413ae9a1db9d80374073b3cc9fb6a", "operation": "LogSoftmax", "operators": [ "LogSoftmax" ], "opset": 19, "optimization": "all", "output_hash": "4f785390cd27f44d94b9870dd77bf2af96019fec70182d75e4c5f2421b6956dc", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.003, "p95_ms": 0.0036582, "repeats": 7, "samples_ms": [ 0.003, 0.003708, 0.003542, 0.002958, 0.002791, 0.003334, 0.002667 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.008584, "p95_ms": 0.0094171, "repeats": 7, "samples_ms": [ 0.009667, 0.008834, 0.008584, 0.008458, 0.0085, 0.008584, 0.00825 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-06, "rtol": 1e-05 }, "variant": "constant", "warnings": [] }, { "artifacts": { "bundle": "observations/2ba076747fa544bbea574a7a38a8115274a92eec9c464e30b05aebf7cc56b92e/reproducer.zip", "bundle_sha256": "6ce2f645e3fab905f39e1ce9883b0c9ee77d0ff9c48baf12b811d378b43041cc", "case": "artifacts/12990c39730af0d8399d7b2b94c3e0b186a85075838e7dfcdad80ac7a08e8ddd/case.json", "inputs": "artifacts/12990c39730af0d8399d7b2b94c3e0b186a85075838e7dfcdad80ac7a08e8ddd/inputs.npz", "manifest": "artifacts/12990c39730af0d8399d7b2b94c3e0b186a85075838e7dfcdad80ac7a08e8ddd/manifest.json", "model": "artifacts/12990c39730af0d8399d7b2b94c3e0b186a85075838e7dfcdad80ac7a08e8ddd/model.onnx", "outputs": "observations/2ba076747fa544bbea574a7a38a8115274a92eec9c464e30b05aebf7cc56b92e/outputs.npz" }, "candidate_group": "39719b3019cdeb423ee47d31af176e009a2dd30eac0a5329f188c48390331561", "case_id": "12990c39730af0d8399d7b2b94c3e0b186a85075838e7dfcdad80ac7a08e8ddd", "category": "softmax", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "backend": "reference", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 1.4741030618381501e-08, "max_budget_ratio": 0.0013400936925801364, "max_rel": 1.4741030618381501e-08, "max_ulp": 66387700, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 17 ], "backend": "reference", "elements": 136, "expected_dtype": "float32", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "backend": "ort", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 1.4741030618381501e-08, "max_budget_ratio": 0.0013400936925801364, "max_rel": 1.4741030618381501e-08, "max_ulp": 66387700, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 17 ], "backend": "ort", "elements": 136, "expected_dtype": "float32", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } } ], "comparison": { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, "comparison_key": "daea044fa2a2c4dab19930d522569981964d0012736806be67fa68ff93afec0e", "contract": "log_probability", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "float32", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "LogSoftmax/constant/float32/4x17/seed-20260906", "observation_id": "2ba076747fa544bbea574a7a38a8115274a92eec9c464e30b05aebf7cc56b92e", "operation": "LogSoftmax", "operators": [ "LogSoftmax" ], "opset": 19, "optimization": "disabled", "output_hash": "4f785390cd27f44d94b9870dd77bf2af96019fec70182d75e4c5f2421b6956dc", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.003167, "p95_ms": 0.0033623999999999998, "repeats": 7, "samples_ms": [ 0.003375, 0.003333, 0.003208, 0.003167, 0.003084, 0.00275, 0.002708 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.00875, "p95_ms": 0.009517, "repeats": 7, "samples_ms": [ 0.009667, 0.009167, 0.008833, 0.008708, 0.008375, 0.00875, 0.008375 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-06, "rtol": 1e-05 }, "variant": "constant", "warnings": [] }, { "artifacts": { "bundle": "observations/0253ae823c2a1741c2186984179c7700ffd05abe1e217fe5cf0dd3321378fb5d/reproducer.zip", "bundle_sha256": "5913ef5d85ebf927408ed1227983563f6c0202124f8031c2441a8753ccdce38e", "case": "artifacts/e2df2c5b49cb05e360eaf5eb58b7233088d0ef6e26d3c5f0aea3b20a4a5261cf/case.json", "inputs": "artifacts/e2df2c5b49cb05e360eaf5eb58b7233088d0ef6e26d3c5f0aea3b20a4a5261cf/inputs.npz", "manifest": "artifacts/e2df2c5b49cb05e360eaf5eb58b7233088d0ef6e26d3c5f0aea3b20a4a5261cf/manifest.json", "model": "artifacts/e2df2c5b49cb05e360eaf5eb58b7233088d0ef6e26d3c5f0aea3b20a4a5261cf/model.onnx", "outputs": "observations/0253ae823c2a1741c2186984179c7700ffd05abe1e217fe5cf0dd3321378fb5d/outputs.npz" }, "candidate_group": "61d693d8319e23ff0433597046524c27d42acaefdcf3437a3c8f246562725d6b", "case_id": "e2df2c5b49cb05e360eaf5eb58b7233088d0ef6e26d3c5f0aea3b20a4a5261cf", "category": "softmax", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "backend": "reference", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 1.1102230246251565e-16, "max_budget_ratio": 0.00010092936587501424, "max_rel": 1.1102230246251565e-16, "max_ulp": 1, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 8, 17 ], "backend": "reference", "elements": 136, "expected_dtype": "float64", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "backend": "ort", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 1.1102230246251565e-16, "max_budget_ratio": 0.00010092936587501424, "max_rel": 1.1102230246251565e-16, "max_ulp": 1, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 8, 17 ], "backend": "ort", "elements": 136, "expected_dtype": "float64", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } } ], "comparison": { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, "comparison_key": "dccfb3be9c8c9364eba2ef97119d202ece7b8b79eaa69b11d76329a6c9172c83", "contract": "log_probability", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "float64", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "LogSoftmax/constant/float64/4x17/seed-20260906", "observation_id": "0253ae823c2a1741c2186984179c7700ffd05abe1e217fe5cf0dd3321378fb5d", "operation": "LogSoftmax", "operators": [ "LogSoftmax" ], "opset": 19, "optimization": "all", "output_hash": "6ae5338ef6726621a195e25d795ec9fe9542347d691c7eceb1c77e2d943bd725", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.003167, "p95_ms": 0.0036087999999999997, "repeats": 7, "samples_ms": [ 0.003709, 0.003375, 0.003208, 0.003167, 0.003, 0.002958, 0.002875 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.008833, "p95_ms": 0.009720699999999999, "repeats": 7, "samples_ms": [ 0.009958, 0.009167, 0.008833, 0.008708, 0.008584, 0.008917, 0.0085 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-13, "rtol": 1e-12 }, "variant": "constant", "warnings": [] }, { "artifacts": { "bundle": "observations/e4eda433c47988c7d06d92d7bf08bdf3ffc31291965e8ea7d71c19572d6e74c9/reproducer.zip", "bundle_sha256": "876cc0b04c6d388c8c0222f71e0ec0f7c4fea8e1a05d9374c872cac940e8cc67", "case": "artifacts/e2df2c5b49cb05e360eaf5eb58b7233088d0ef6e26d3c5f0aea3b20a4a5261cf/case.json", "inputs": "artifacts/e2df2c5b49cb05e360eaf5eb58b7233088d0ef6e26d3c5f0aea3b20a4a5261cf/inputs.npz", "manifest": "artifacts/e2df2c5b49cb05e360eaf5eb58b7233088d0ef6e26d3c5f0aea3b20a4a5261cf/manifest.json", "model": "artifacts/e2df2c5b49cb05e360eaf5eb58b7233088d0ef6e26d3c5f0aea3b20a4a5261cf/model.onnx", "outputs": "observations/e4eda433c47988c7d06d92d7bf08bdf3ffc31291965e8ea7d71c19572d6e74c9/outputs.npz" }, "candidate_group": "61d693d8319e23ff0433597046524c27d42acaefdcf3437a3c8f246562725d6b", "case_id": "e2df2c5b49cb05e360eaf5eb58b7233088d0ef6e26d3c5f0aea3b20a4a5261cf", "category": "softmax", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "backend": "reference", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 1.1102230246251565e-16, "max_budget_ratio": 0.00010092936587501424, "max_rel": 1.1102230246251565e-16, "max_ulp": 1, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 8, 17 ], "backend": "reference", "elements": 136, "expected_dtype": "float64", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "backend": "ort", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 1.1102230246251565e-16, "max_budget_ratio": 0.00010092936587501424, "max_rel": 1.1102230246251565e-16, "max_ulp": 1, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 8, 17 ], "backend": "ort", "elements": 136, "expected_dtype": "float64", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } } ], "comparison": { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, "comparison_key": "9b17a95b057448747b5c34364a5cf8cbec6d9064d1992c3e2bda1589bc1e9712", "contract": "log_probability", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "float64", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "LogSoftmax/constant/float64/4x17/seed-20260906", "observation_id": "e4eda433c47988c7d06d92d7bf08bdf3ffc31291965e8ea7d71c19572d6e74c9", "operation": "LogSoftmax", "operators": [ "LogSoftmax" ], "opset": 19, "optimization": "disabled", "output_hash": "6ae5338ef6726621a195e25d795ec9fe9542347d691c7eceb1c77e2d943bd725", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.003167, "p95_ms": 0.0035499999999999998, "repeats": 7, "samples_ms": [ 0.003375, 0.003625, 0.003167, 0.003375, 0.003167, 0.003, 0.002792 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.008875, "p95_ms": 0.009474999999999999, "repeats": 7, "samples_ms": [ 0.009625, 0.009125, 0.008875, 0.008625, 0.0085, 0.008875, 0.008417 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-13, "rtol": 1e-12 }, "variant": "constant", "warnings": [] }, { "artifacts": { "bundle": "observations/c7a4552d3ab02a035503e6fd956d8a3348663d1b5e6c251f38b92e3cf046b3b8/reproducer.zip", "bundle_sha256": "9f19b74ca58465d930adee3fd9b6ffd0f8200b7c2dc1c7f6b0dcce35c7ee77b0", "case": "artifacts/fba350a87ca2e4b7a81f55d0d14033fc281d40e2979b6c6f9d25592428d27eb2/case.json", "inputs": "artifacts/fba350a87ca2e4b7a81f55d0d14033fc281d40e2979b6c6f9d25592428d27eb2/inputs.npz", "manifest": "artifacts/fba350a87ca2e4b7a81f55d0d14033fc281d40e2979b6c6f9d25592428d27eb2/manifest.json", "model": "artifacts/fba350a87ca2e4b7a81f55d0d14033fc281d40e2979b6c6f9d25592428d27eb2/model.onnx", "outputs": "observations/c7a4552d3ab02a035503e6fd956d8a3348663d1b5e6c251f38b92e3cf046b3b8/outputs.npz" }, "candidate_group": "17f28606a4d6bc76adbbf49bf7c70070b364840a20d09e6e9ee9910810622171", "case_id": "fba350a87ca2e4b7a81f55d0d14033fc281d40e2979b6c6f9d25592428d27eb2", "category": "softmax", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "backend": "reference", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 0.00033713016747760616, "max_budget_ratio": 0.061296394086837486, "max_rel": 0.00033713016747760616, "max_ulp": 3036598593255, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 8, 17 ], "backend": "reference", "elements": 136, "expected_dtype": "float16", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.001953125, "max_budget_ratio": 0.14072614691809737, "max_rel": 0.00072992700729927, "max_ulp": 1, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "backend": "ort", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 0.0008692734914177436, "max_budget_ratio": 0.158049725712317, "max_rel": 0.0008692734914177436, "max_ulp": 5651390359436, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 8, 17 ], "backend": "ort", "elements": 136, "expected_dtype": "float16", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } } ], "comparison": { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.001953125, "max_budget_ratio": 0.13457139012245997, "max_rel": 0.0006968641114982578, "max_ulp": 1, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, "comparison_key": "7cd30b777a6f18dcae4cbba1889768403fd0270ba1e6e70c4e1b9441ed4b1eba", "contract": "log_probability", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "float16", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "LogSoftmax/large_offset/float16/4x17/seed-20260906", "observation_id": "c7a4552d3ab02a035503e6fd956d8a3348663d1b5e6c251f38b92e3cf046b3b8", "operation": "LogSoftmax", "operators": [ "LogSoftmax" ], "opset": 19, "optimization": "all", "output_hash": "717a422ed255f8ec8311e06e34e5e45df720879c4c36b92f0c9cc2a3b5245a90", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.005, "p95_ms": 0.005424999999999999, "repeats": 7, "samples_ms": [ 0.0055, 0.005209, 0.005, 0.00525, 0.004875, 0.004334, 0.004375 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.009375, "p95_ms": 0.0104501, "repeats": 7, "samples_ms": [ 0.010625, 0.010042, 0.009666, 0.009375, 0.009291, 0.009292, 0.009083 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 0.0005, "rtol": 0.005 }, "variant": "large_offset", "warnings": [] }, { "artifacts": { "bundle": "observations/6db01999d9c9c12b2b730925bd45430d7bf571b0931c07fc42bc4eeec8d315fe/reproducer.zip", "bundle_sha256": "bdeb60b40387e069e59bfb62a1e9a3b173c21d9a18804b4beebd776ac49db26a", "case": "artifacts/fba350a87ca2e4b7a81f55d0d14033fc281d40e2979b6c6f9d25592428d27eb2/case.json", "inputs": "artifacts/fba350a87ca2e4b7a81f55d0d14033fc281d40e2979b6c6f9d25592428d27eb2/inputs.npz", "manifest": "artifacts/fba350a87ca2e4b7a81f55d0d14033fc281d40e2979b6c6f9d25592428d27eb2/manifest.json", "model": "artifacts/fba350a87ca2e4b7a81f55d0d14033fc281d40e2979b6c6f9d25592428d27eb2/model.onnx", "outputs": "observations/6db01999d9c9c12b2b730925bd45430d7bf571b0931c07fc42bc4eeec8d315fe/outputs.npz" }, "candidate_group": "17f28606a4d6bc76adbbf49bf7c70070b364840a20d09e6e9ee9910810622171", "case_id": "fba350a87ca2e4b7a81f55d0d14033fc281d40e2979b6c6f9d25592428d27eb2", "category": "softmax", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "backend": "reference", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 0.00033713016747760616, "max_budget_ratio": 0.061296394086837486, "max_rel": 0.00033713016747760616, "max_ulp": 3036598593255, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 8, 17 ], "backend": "reference", "elements": 136, "expected_dtype": "float16", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.001953125, "max_budget_ratio": 0.14072614691809737, "max_rel": 0.00072992700729927, "max_ulp": 1, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "backend": "ort", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 0.0008692734914177436, "max_budget_ratio": 0.158049725712317, "max_rel": 0.0008692734914177436, "max_ulp": 5651390359436, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 8, 17 ], "backend": "ort", "elements": 136, "expected_dtype": "float16", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } } ], "comparison": { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.001953125, "max_budget_ratio": 0.13457139012245997, "max_rel": 0.0006968641114982578, "max_ulp": 1, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, "comparison_key": "c433f7baa3f01cef80c6c46c5470432de18b11b1222cbe630df694f90b3f8122", "contract": "log_probability", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "float16", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "LogSoftmax/large_offset/float16/4x17/seed-20260906", "observation_id": "6db01999d9c9c12b2b730925bd45430d7bf571b0931c07fc42bc4eeec8d315fe", "operation": "LogSoftmax", "operators": [ "LogSoftmax" ], "opset": 19, "optimization": "disabled", "output_hash": "717a422ed255f8ec8311e06e34e5e45df720879c4c36b92f0c9cc2a3b5245a90", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.005042, "p95_ms": 0.0054167, "repeats": 7, "samples_ms": [ 0.005417, 0.005416, 0.005042, 0.005042, 0.004875, 0.004834, 0.004917 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.009292, "p95_ms": 0.0103751, "repeats": 7, "samples_ms": [ 0.010625, 0.009792, 0.009541, 0.00925, 0.009292, 0.00925, 0.009125 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 0.0005, "rtol": 0.005 }, "variant": "large_offset", "warnings": [] }, { "artifacts": { "bundle": "observations/033f524e3c6bc68ddb2987a6c65f48a164d4616cc6c5ff33a4ce3fbfdaef493f/reproducer.zip", "bundle_sha256": "189482a63cda9a37e66325d853e6105b552390213960a608552ac6dc0619f3d4", "case": "artifacts/223ca71fcc677dd37f0639f91efcd8ed816b6283867cb6725abe69c30ce0156f/case.json", "inputs": "artifacts/223ca71fcc677dd37f0639f91efcd8ed816b6283867cb6725abe69c30ce0156f/inputs.npz", "manifest": "artifacts/223ca71fcc677dd37f0639f91efcd8ed816b6283867cb6725abe69c30ce0156f/manifest.json", "model": "artifacts/223ca71fcc677dd37f0639f91efcd8ed816b6283867cb6725abe69c30ce0156f/model.onnx", "outputs": "observations/033f524e3c6bc68ddb2987a6c65f48a164d4616cc6c5ff33a4ce3fbfdaef493f/outputs.npz" }, "candidate_group": "c4a6e486e4116b12c5f12b619cab500fab9cae055974a300a30a9f71631f6b32", "case_id": "223ca71fcc677dd37f0639f91efcd8ed816b6283867cb6725abe69c30ce0156f", "category": "softmax", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "backend": "reference", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 7.359833054110254e-08, "max_budget_ratio": 0.006690757321918412, "max_rel": 7.359833054110254e-08, "max_ulp": 331457414, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 17 ], "backend": "reference", "elements": 136, "expected_dtype": "float32", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 2.384185791015625e-07, "max_budget_ratio": 0.008932110458336598, "max_rel": 9.279767545534892e-08, "max_ulp": 1, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "backend": "ort", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 8.98152321404666e-08, "max_budget_ratio": 0.00816502110367878, "max_rel": 8.98152321404666e-08, "max_ulp": 672251616, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 17 ], "backend": "ort", "elements": 136, "expected_dtype": "float32", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } } ], "comparison": { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 2.384185791015625e-07, "max_budget_ratio": 0.008565462066335152, "max_rel": 8.884653475858575e-08, "max_ulp": 1, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, "comparison_key": "3bdfafb97346cd8998c777111813085ea89625681f53b39a633568b8e0e914cd", "contract": "log_probability", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "float32", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "LogSoftmax/large_offset/float32/4x17/seed-20260906", "observation_id": "033f524e3c6bc68ddb2987a6c65f48a164d4616cc6c5ff33a4ce3fbfdaef493f", "operation": "LogSoftmax", "operators": [ "LogSoftmax" ], "opset": 19, "optimization": "all", "output_hash": "eea329de5eb2d0d1254b7200929c46c0087d6eb267c3e39bea1b910f2695db08", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.002875, "p95_ms": 0.0032792999999999997, "repeats": 7, "samples_ms": [ 0.002958, 0.003417, 0.002958, 0.002875, 0.002791, 0.002875, 0.002708 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.008333, "p95_ms": 0.0093749, "repeats": 7, "samples_ms": [ 0.0095, 0.009083, 0.008542, 0.008333, 0.008083, 0.008333, 0.008166 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-06, "rtol": 1e-05 }, "variant": "large_offset", "warnings": [] }, { "artifacts": { "bundle": "observations/bf25604aeaca4f97f1f1c611210c76246d760ab7581dfcdd99fb5101bb416d35/reproducer.zip", "bundle_sha256": "9f0304f060604173f5b8919e3f8de1aa744665bc177273eb8e9addf124122395", "case": "artifacts/223ca71fcc677dd37f0639f91efcd8ed816b6283867cb6725abe69c30ce0156f/case.json", "inputs": "artifacts/223ca71fcc677dd37f0639f91efcd8ed816b6283867cb6725abe69c30ce0156f/inputs.npz", "manifest": "artifacts/223ca71fcc677dd37f0639f91efcd8ed816b6283867cb6725abe69c30ce0156f/manifest.json", "model": "artifacts/223ca71fcc677dd37f0639f91efcd8ed816b6283867cb6725abe69c30ce0156f/model.onnx", "outputs": "observations/bf25604aeaca4f97f1f1c611210c76246d760ab7581dfcdd99fb5101bb416d35/outputs.npz" }, "candidate_group": "c4a6e486e4116b12c5f12b619cab500fab9cae055974a300a30a9f71631f6b32", "case_id": "223ca71fcc677dd37f0639f91efcd8ed816b6283867cb6725abe69c30ce0156f", "category": "softmax", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "backend": "reference", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 7.359833054110254e-08, "max_budget_ratio": 0.006690757321918412, "max_rel": 7.359833054110254e-08, "max_ulp": 331457414, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 17 ], "backend": "reference", "elements": 136, "expected_dtype": "float32", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 2.384185791015625e-07, "max_budget_ratio": 0.008932110458336598, "max_rel": 9.279767545534892e-08, "max_ulp": 1, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "backend": "ort", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 8.98152321404666e-08, "max_budget_ratio": 0.00816502110367878, "max_rel": 8.98152321404666e-08, "max_ulp": 672251616, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 17 ], "backend": "ort", "elements": 136, "expected_dtype": "float32", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } } ], "comparison": { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 2.384185791015625e-07, "max_budget_ratio": 0.008565462066335152, "max_rel": 8.884653475858575e-08, "max_ulp": 1, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, "comparison_key": "7bc7f69a93ec79fc0d6959439fbc8553130a13ffca95987c53c5781f8bb7a30d", "contract": "log_probability", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "float32", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "LogSoftmax/large_offset/float32/4x17/seed-20260906", "observation_id": "bf25604aeaca4f97f1f1c611210c76246d760ab7581dfcdd99fb5101bb416d35", "operation": "LogSoftmax", "operators": [ "LogSoftmax" ], "opset": 19, "optimization": "disabled", "output_hash": "eea329de5eb2d0d1254b7200929c46c0087d6eb267c3e39bea1b910f2695db08", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.008125, "p95_ms": 0.008812599999999999, "repeats": 7, "samples_ms": [ 0.008667, 0.008875, 0.0085, 0.008125, 0.007917, 0.007833, 0.007667 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.020584, "p95_ms": 0.022137399999999998, "repeats": 7, "samples_ms": [ 0.022375, 0.021583, 0.020875, 0.020167, 0.020208, 0.020584, 0.020125 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-06, "rtol": 1e-05 }, "variant": "large_offset", "warnings": [] }, { "artifacts": { "bundle": "observations/d0c74fbf74e25c923b5d9889c3dbe030006a63d030fce2226f61125437bb0ba3/reproducer.zip", "bundle_sha256": "3c1730906878922e3fe091755ca75777308d9574ed30d56c23ca98ed725c846d", "case": "artifacts/d813fed155d60e1d98e3314fd012a8315222cb75f8b6610c01a5c9966e973a2b/case.json", "inputs": "artifacts/d813fed155d60e1d98e3314fd012a8315222cb75f8b6610c01a5c9966e973a2b/inputs.npz", "manifest": "artifacts/d813fed155d60e1d98e3314fd012a8315222cb75f8b6610c01a5c9966e973a2b/manifest.json", "model": "artifacts/d813fed155d60e1d98e3314fd012a8315222cb75f8b6610c01a5c9966e973a2b/model.onnx", "outputs": "observations/d0c74fbf74e25c923b5d9889c3dbe030006a63d030fce2226f61125437bb0ba3/outputs.npz" }, "candidate_group": "5eb626ab5f17c5c921112ef67917dc96bd221e5806e584536772823e10a4a6f3", "case_id": "d813fed155d60e1d98e3314fd012a8315222cb75f8b6610c01a5c9966e973a2b", "category": "softmax", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "backend": "reference", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 2.220446049250313e-16, "max_budget_ratio": 0.00020185873175002848, "max_rel": 2.220446049250313e-16, "max_ulp": 1, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 8, 17 ], "backend": "reference", "elements": 136, "expected_dtype": "float64", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 4.440892098500626e-16, "max_budget_ratio": 0.00015645269279148988, "max_rel": 1.6216579613217228e-16, "max_ulp": 1, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "backend": "ort", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 2.220446049250313e-16, "max_budget_ratio": 0.00020185873175002848, "max_rel": 2.220446049250313e-16, "max_ulp": 2, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 8, 17 ], "backend": "ort", "elements": 136, "expected_dtype": "float64", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 4.440892098500626e-16, "max_budget_ratio": 0.00016397075130063183, "max_rel": 1.7025714403490451e-16, "max_ulp": 1, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } } ], "comparison": { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 4.440892098500626e-16, "max_budget_ratio": 0.00015645269279148988, "max_rel": 1.6216579613217228e-16, "max_ulp": 1, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, "comparison_key": "94459997145008e2eeb669d02fd09ec63f744cd0f04a0179e8425eb733836434", "contract": "log_probability", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "float64", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "LogSoftmax/large_offset/float64/4x17/seed-20260906", "observation_id": "d0c74fbf74e25c923b5d9889c3dbe030006a63d030fce2226f61125437bb0ba3", "operation": "LogSoftmax", "operators": [ "LogSoftmax" ], "opset": 19, "optimization": "all", "output_hash": "b70451bd909fd112677baa7e7e3465b25f7d182580a19ab5a1d9aa5f7d6a7302", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.003417, "p95_ms": 0.0038626, "repeats": 7, "samples_ms": [ 0.003542, 0.004, 0.003458, 0.003417, 0.003375, 0.003083, 0.003166 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.008666, "p95_ms": 0.0096206, "repeats": 7, "samples_ms": [ 0.009833, 0.009125, 0.008792, 0.008666, 0.0085, 0.008625, 0.008417 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-13, "rtol": 1e-12 }, "variant": "large_offset", "warnings": [] }, { "artifacts": { "bundle": "observations/dc7c13a654ffabf51869b9e3fec9a8549507cb4d87bbc9ca20e60a61bb01d436/reproducer.zip", "bundle_sha256": "f6b9ed66f6a0588765f37e162e49e8be077f3b45accdce98d7d2faf50c8401b0", "case": "artifacts/d813fed155d60e1d98e3314fd012a8315222cb75f8b6610c01a5c9966e973a2b/case.json", "inputs": "artifacts/d813fed155d60e1d98e3314fd012a8315222cb75f8b6610c01a5c9966e973a2b/inputs.npz", "manifest": "artifacts/d813fed155d60e1d98e3314fd012a8315222cb75f8b6610c01a5c9966e973a2b/manifest.json", "model": "artifacts/d813fed155d60e1d98e3314fd012a8315222cb75f8b6610c01a5c9966e973a2b/model.onnx", "outputs": "observations/dc7c13a654ffabf51869b9e3fec9a8549507cb4d87bbc9ca20e60a61bb01d436/outputs.npz" }, "candidate_group": "5eb626ab5f17c5c921112ef67917dc96bd221e5806e584536772823e10a4a6f3", "case_id": "d813fed155d60e1d98e3314fd012a8315222cb75f8b6610c01a5c9966e973a2b", "category": "softmax", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "backend": "reference", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 2.220446049250313e-16, "max_budget_ratio": 0.00020185873175002848, "max_rel": 2.220446049250313e-16, "max_ulp": 1, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 8, 17 ], "backend": "reference", "elements": 136, "expected_dtype": "float64", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 4.440892098500626e-16, "max_budget_ratio": 0.00015645269279148988, "max_rel": 1.6216579613217228e-16, "max_ulp": 1, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "backend": "ort", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 2.220446049250313e-16, "max_budget_ratio": 0.00020185873175002848, "max_rel": 2.220446049250313e-16, "max_ulp": 2, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 8, 17 ], "backend": "ort", "elements": 136, "expected_dtype": "float64", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 4.440892098500626e-16, "max_budget_ratio": 0.00016397075130063183, "max_rel": 1.7025714403490451e-16, "max_ulp": 1, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } } ], "comparison": { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 4.440892098500626e-16, "max_budget_ratio": 0.00015645269279148988, "max_rel": 1.6216579613217228e-16, "max_ulp": 1, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, "comparison_key": "bba4d41d4779d762c575ea21f767ee3aa6b8277f1df05b638baea75b010c5463", "contract": "log_probability", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "float64", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "LogSoftmax/large_offset/float64/4x17/seed-20260906", "observation_id": "dc7c13a654ffabf51869b9e3fec9a8549507cb4d87bbc9ca20e60a61bb01d436", "operation": "LogSoftmax", "operators": [ "LogSoftmax" ], "opset": 19, "optimization": "disabled", "output_hash": "b70451bd909fd112677baa7e7e3465b25f7d182580a19ab5a1d9aa5f7d6a7302", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.003208, "p95_ms": 0.0037413999999999998, "repeats": 7, "samples_ms": [ 0.00325, 0.003916, 0.003334, 0.003208, 0.003208, 0.003125, 0.003208 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.008834, "p95_ms": 0.0098625, "repeats": 7, "samples_ms": [ 0.010125, 0.00925, 0.008875, 0.008834, 0.00875, 0.00875, 0.008542 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-13, "rtol": 1e-12 }, "variant": "large_offset", "warnings": [] }, { "artifacts": { "bundle": "observations/ade04741e3b9ac5c0a9b2d319b674e6ac03ddfec40d15466b3676471a76e863b/reproducer.zip", "bundle_sha256": "7ffa2b71ddab28431f46747677d462537cce019ae6d403a4e0a75f5abb3b721a", "case": "artifacts/d17f5e8372cb240f77968c045384dd952398b2e8427864bd89ec90fa88a462f7/case.json", "inputs": "artifacts/d17f5e8372cb240f77968c045384dd952398b2e8427864bd89ec90fa88a462f7/inputs.npz", "manifest": "artifacts/d17f5e8372cb240f77968c045384dd952398b2e8427864bd89ec90fa88a462f7/manifest.json", "model": "artifacts/d17f5e8372cb240f77968c045384dd952398b2e8427864bd89ec90fa88a462f7/model.onnx", "outputs": "observations/ade04741e3b9ac5c0a9b2d319b674e6ac03ddfec40d15466b3676471a76e863b/outputs.npz" }, "candidate_group": "4de5f7bbdd0e40c9daf9f48cd1fbca0f415710fcf122fb90cf62f6e1c4979245", "case_id": "d17f5e8372cb240f77968c045384dd952398b2e8427864bd89ec90fa88a462f7", "category": "softmax", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "backend": "reference", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 0.0008414401776245528, "max_budget_ratio": 0.15298912320446414, "max_rel": 0.0008414401776245528, "max_ulp": 7579019340809, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 8, 17 ], "backend": "reference", "elements": 136, "expected_dtype": "float16", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.00390625, "max_budget_ratio": 0.17510068289266328, "max_rel": 0.0009165902841429881, "max_ulp": 1, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "backend": "ort", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 0.0003339385721199406, "max_budget_ratio": 0.06071610402180739, "max_rel": 0.0003339385721199406, "max_ulp": 1503925628964, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 8, 17 ], "backend": "ort", "elements": 136, "expected_dtype": "float16", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } } ], "comparison": { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.00390625, "max_budget_ratio": 0.17510068289266328, "max_rel": 0.0009165902841429881, "max_ulp": 1, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, "comparison_key": "3d8a1f53b194e5ad8e8580941fc278117c5332c10022812d216c1145bea85684", "contract": "log_probability", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "float16", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "LogSoftmax/ordinary/float16/4x17/seed-20260906", "observation_id": "ade04741e3b9ac5c0a9b2d319b674e6ac03ddfec40d15466b3676471a76e863b", "operation": "LogSoftmax", "operators": [ "LogSoftmax" ], "opset": 19, "optimization": "all", "output_hash": "916120ca95bebd995714fb8b4ad62e1bc39bd811545c4f282ed938a777fc9565", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.005333, "p95_ms": 0.0061043, "repeats": 7, "samples_ms": [ 0.006167, 0.005958, 0.005375, 0.005333, 0.004583, 0.0045, 0.004208 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.010166, "p95_ms": 0.0107625, "repeats": 7, "samples_ms": [ 0.010875, 0.010334, 0.0105, 0.009875, 0.010166, 0.009792, 0.009625 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 0.0005, "rtol": 0.005 }, "variant": "ordinary", "warnings": [] }, { "artifacts": { "bundle": "observations/fe654a480f157fa79fbdc7a1cde304b1ec8fc2a7dd96af06ba2aa23b6e5cc624/reproducer.zip", "bundle_sha256": "b3dc14d9a59d728d7cf9e2c36f41caf169bc7f6f27d4c9f6e5d163871ace0b79", "case": "artifacts/d17f5e8372cb240f77968c045384dd952398b2e8427864bd89ec90fa88a462f7/case.json", "inputs": "artifacts/d17f5e8372cb240f77968c045384dd952398b2e8427864bd89ec90fa88a462f7/inputs.npz", "manifest": "artifacts/d17f5e8372cb240f77968c045384dd952398b2e8427864bd89ec90fa88a462f7/manifest.json", "model": "artifacts/d17f5e8372cb240f77968c045384dd952398b2e8427864bd89ec90fa88a462f7/model.onnx", "outputs": "observations/fe654a480f157fa79fbdc7a1cde304b1ec8fc2a7dd96af06ba2aa23b6e5cc624/outputs.npz" }, "candidate_group": "4de5f7bbdd0e40c9daf9f48cd1fbca0f415710fcf122fb90cf62f6e1c4979245", "case_id": "d17f5e8372cb240f77968c045384dd952398b2e8427864bd89ec90fa88a462f7", "category": "softmax", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "backend": "reference", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 0.0008414401776245528, "max_budget_ratio": 0.15298912320446414, "max_rel": 0.0008414401776245528, "max_ulp": 7579019340809, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 8, 17 ], "backend": "reference", "elements": 136, "expected_dtype": "float16", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.00390625, "max_budget_ratio": 0.17510068289266328, "max_rel": 0.0009165902841429881, "max_ulp": 1, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "backend": "ort", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 0.0003339385721199406, "max_budget_ratio": 0.06071610402180739, "max_rel": 0.0003339385721199406, "max_ulp": 1503925628964, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 8, 17 ], "backend": "ort", "elements": 136, "expected_dtype": "float16", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } } ], "comparison": { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.00390625, "max_budget_ratio": 0.17510068289266328, "max_rel": 0.0009165902841429881, "max_ulp": 1, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, "comparison_key": "976408c9e2a9944f872b8d8c50febb4a7a1acb285d4334b5977682e41c78aee2", "contract": "log_probability", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "float16", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "LogSoftmax/ordinary/float16/4x17/seed-20260906", "observation_id": "fe654a480f157fa79fbdc7a1cde304b1ec8fc2a7dd96af06ba2aa23b6e5cc624", "operation": "LogSoftmax", "operators": [ "LogSoftmax" ], "opset": 19, "optimization": "disabled", "output_hash": "916120ca95bebd995714fb8b4ad62e1bc39bd811545c4f282ed938a777fc9565", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.005417, "p95_ms": 0.00605, "repeats": 7, "samples_ms": [ 0.006125, 0.005875, 0.005417, 0.005167, 0.005417, 0.00475, 0.0045 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.009708, "p95_ms": 0.0106465, "repeats": 7, "samples_ms": [ 0.010834, 0.010209, 0.009958, 0.009583, 0.009542, 0.009708, 0.009541 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 0.0005, "rtol": 0.005 }, "variant": "ordinary", "warnings": [] }, { "artifacts": { "bundle": "observations/39ee2bd932bb56e1b4aa7ce242d3f0b9705bc62290d25762452b543bbeb64a9e/reproducer.zip", "bundle_sha256": "dc3dcda5e998fb76709e417febf0a819aba935f9c8f545f7cfc49f767f9f54dd", "case": "artifacts/482121aa0ca904fde0aa01acde207bc3ba4d1a75ff9fb055c95cc105157fb4ca/case.json", "inputs": "artifacts/482121aa0ca904fde0aa01acde207bc3ba4d1a75ff9fb055c95cc105157fb4ca/inputs.npz", "manifest": "artifacts/482121aa0ca904fde0aa01acde207bc3ba4d1a75ff9fb055c95cc105157fb4ca/manifest.json", "model": "artifacts/482121aa0ca904fde0aa01acde207bc3ba4d1a75ff9fb055c95cc105157fb4ca/model.onnx", "outputs": "observations/39ee2bd932bb56e1b4aa7ce242d3f0b9705bc62290d25762452b543bbeb64a9e/outputs.npz" }, "candidate_group": "9b939978d2359f6a0e73e1c5ac8e8239295597236bd647242ff2efc64868d768", "case_id": "482121aa0ca904fde0aa01acde207bc3ba4d1a75ff9fb055c95cc105157fb4ca", "category": "softmax", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "backend": "reference", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 5.308024686456747e-08, "max_budget_ratio": 0.004825476987687951, "max_rel": 5.308024686456747e-08, "max_ulp": 239052180, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 17 ], "backend": "reference", "elements": 136, "expected_dtype": "float32", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 4.76837158203125e-07, "max_budget_ratio": 0.011495151080885608, "max_rel": 1.1779111042268751e-07, "max_ulp": 1, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "backend": "ort", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 7.477895525020983e-08, "max_budget_ratio": 0.006798086840928166, "max_rel": 7.477895525020983e-08, "max_ulp": 673548950, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 17 ], "backend": "ort", "elements": 136, "expected_dtype": "float32", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } } ], "comparison": { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 4.76837158203125e-07, "max_budget_ratio": 0.011495151080885608, "max_rel": 1.1779111042268751e-07, "max_ulp": 1, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, "comparison_key": "8e0abb3472b3794e6808a1a926942144b9bc8c1a7c900979f41be5042211aa61", "contract": "log_probability", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "float32", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "LogSoftmax/ordinary/float32/4x17/seed-20260906", "observation_id": "39ee2bd932bb56e1b4aa7ce242d3f0b9705bc62290d25762452b543bbeb64a9e", "operation": "LogSoftmax", "operators": [ "LogSoftmax" ], "opset": 19, "optimization": "all", "output_hash": "c2aee4186f863fe6fc5043e61b0f15a6d041f2a184efbe805d513e7619a8833f", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.002959, "p95_ms": 0.0031999, "repeats": 7, "samples_ms": [ 0.002875, 0.00325, 0.003042, 0.002917, 0.003083, 0.002959, 0.002875 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.008708, "p95_ms": 0.0094624, "repeats": 7, "samples_ms": [ 0.009625, 0.009083, 0.009, 0.008708, 0.008583, 0.008667, 0.0085 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-06, "rtol": 1e-05 }, "variant": "ordinary", "warnings": [] }, { "artifacts": { "bundle": "observations/e4cf080c32d6b87601053983c5e23d9c81987b436b82be64697cd28f4a0b2047/reproducer.zip", "bundle_sha256": "51882f9928ca6b7b253977422e92649954a500300f6e3b4126f7f68b646c5806", "case": "artifacts/482121aa0ca904fde0aa01acde207bc3ba4d1a75ff9fb055c95cc105157fb4ca/case.json", "inputs": "artifacts/482121aa0ca904fde0aa01acde207bc3ba4d1a75ff9fb055c95cc105157fb4ca/inputs.npz", "manifest": "artifacts/482121aa0ca904fde0aa01acde207bc3ba4d1a75ff9fb055c95cc105157fb4ca/manifest.json", "model": "artifacts/482121aa0ca904fde0aa01acde207bc3ba4d1a75ff9fb055c95cc105157fb4ca/model.onnx", "outputs": "observations/e4cf080c32d6b87601053983c5e23d9c81987b436b82be64697cd28f4a0b2047/outputs.npz" }, "candidate_group": "9b939978d2359f6a0e73e1c5ac8e8239295597236bd647242ff2efc64868d768", "case_id": "482121aa0ca904fde0aa01acde207bc3ba4d1a75ff9fb055c95cc105157fb4ca", "category": "softmax", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "backend": "reference", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 5.308024686456747e-08, "max_budget_ratio": 0.004825476987687951, "max_rel": 5.308024686456747e-08, "max_ulp": 239052180, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 17 ], "backend": "reference", "elements": 136, "expected_dtype": "float32", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 4.76837158203125e-07, "max_budget_ratio": 0.011495151080885608, "max_rel": 1.1779111042268751e-07, "max_ulp": 1, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "backend": "ort", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 7.477895525020983e-08, "max_budget_ratio": 0.006798086840928166, "max_rel": 7.477895525020983e-08, "max_ulp": 673548950, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 17 ], "backend": "ort", "elements": 136, "expected_dtype": "float32", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } } ], "comparison": { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 4.76837158203125e-07, "max_budget_ratio": 0.011495151080885608, "max_rel": 1.1779111042268751e-07, "max_ulp": 1, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, "comparison_key": "0f0371012eb7b72b1af5b490221c1061025c243ab3baad608ed052a17bc1904d", "contract": "log_probability", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "float32", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "LogSoftmax/ordinary/float32/4x17/seed-20260906", "observation_id": "e4cf080c32d6b87601053983c5e23d9c81987b436b82be64697cd28f4a0b2047", "operation": "LogSoftmax", "operators": [ "LogSoftmax" ], "opset": 19, "optimization": "disabled", "output_hash": "c2aee4186f863fe6fc5043e61b0f15a6d041f2a184efbe805d513e7619a8833f", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.003083, "p95_ms": 0.0035212999999999998, "repeats": 7, "samples_ms": [ 0.003166, 0.003584, 0.003375, 0.003083, 0.002959, 0.002708, 0.002791 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.008959, "p95_ms": 0.0097544, "repeats": 7, "samples_ms": [ 0.009917, 0.009334, 0.008959, 0.008917, 0.008833, 0.009375, 0.008709 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-06, "rtol": 1e-05 }, "variant": "ordinary", "warnings": [] }, { "artifacts": { "bundle": "observations/8af6c5c09dd1bc04b6e2ee98116bef3ab02c00427f7981907c87a04bd491af06/reproducer.zip", "bundle_sha256": "bbb1bd38038fac31fb5bd6bf2b7294db637dfc078df80dcb2a516329c258959e", "case": "artifacts/b3e7f630ac8ff0f5353c37f41cc3ea6d9a33ce34197c5a371304fc2665deff11/case.json", "inputs": "artifacts/b3e7f630ac8ff0f5353c37f41cc3ea6d9a33ce34197c5a371304fc2665deff11/inputs.npz", "manifest": "artifacts/b3e7f630ac8ff0f5353c37f41cc3ea6d9a33ce34197c5a371304fc2665deff11/manifest.json", "model": "artifacts/b3e7f630ac8ff0f5353c37f41cc3ea6d9a33ce34197c5a371304fc2665deff11/model.onnx", "outputs": "observations/8af6c5c09dd1bc04b6e2ee98116bef3ab02c00427f7981907c87a04bd491af06/outputs.npz" }, "candidate_group": "a4c73789756cf3e2fb2342a6380f0629bb720f3b128c89e26ed3afd2e4951bc6", "case_id": "b3e7f630ac8ff0f5353c37f41cc3ea6d9a33ce34197c5a371304fc2665deff11", "category": "softmax", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "backend": "reference", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 2.220446049250313e-16, "max_budget_ratio": 0.00020185873175002848, "max_rel": 2.220446049250313e-16, "max_ulp": 1, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 8, 17 ], "backend": "reference", "elements": 136, "expected_dtype": "float64", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 8.881784197001252e-16, "max_budget_ratio": 0.0002100763289896815, "max_rel": 2.2050742937458016e-16, "max_ulp": 1, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "backend": "ort", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 4.440892098500626e-16, "max_budget_ratio": 0.00040371746350005696, "max_rel": 4.440892098500626e-16, "max_ulp": 2, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 8.881784197001252e-16, "max_budget_ratio": 0.00029814837700225334, "max_rel": 3.0850441453380463e-16, "max_ulp": 2, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 8, 17 ], "backend": "ort", "elements": 136, "expected_dtype": "float64", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 8.881784197001252e-16, "max_budget_ratio": 0.00018737145188598515, "max_rel": 1.9140945445281596e-16, "max_ulp": 1, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } } ], "comparison": { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 8.881784197001252e-16, "max_budget_ratio": 0.0002100763289896815, "max_rel": 2.2050742937458016e-16, "max_ulp": 1, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, "comparison_key": "7250e648e7cd2b0436adb4583368a2e202e645b45d9908701956ad35c386a9ff", "contract": "log_probability", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "float64", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "LogSoftmax/ordinary/float64/4x17/seed-20260906", "observation_id": "8af6c5c09dd1bc04b6e2ee98116bef3ab02c00427f7981907c87a04bd491af06", "operation": "LogSoftmax", "operators": [ "LogSoftmax" ], "opset": 19, "optimization": "all", "output_hash": "a82d381d0eec66bfcbfa30052aa3b2e1348a98e6274715ed18c09d8024993436", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.003333, "p95_ms": 0.003924899999999999, "repeats": 7, "samples_ms": [ 0.003375, 0.004125, 0.003333, 0.003458, 0.00325, 0.003125, 0.002958 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.008625, "p95_ms": 0.009817, "repeats": 7, "samples_ms": [ 0.010042, 0.009292, 0.008625, 0.008542, 0.008583, 0.008667, 0.0085 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-13, "rtol": 1e-12 }, "variant": "ordinary", "warnings": [] }, { "artifacts": { "bundle": "observations/81748235eb33a34124aad679391361848fb6eed683ceffae2bcbfb77f06341b9/reproducer.zip", "bundle_sha256": "16a21b655f5b88fe8ddee8ff3f29a72cbf9ea5c3bc200cf09dc944f202fada58", "case": "artifacts/b3e7f630ac8ff0f5353c37f41cc3ea6d9a33ce34197c5a371304fc2665deff11/case.json", "inputs": "artifacts/b3e7f630ac8ff0f5353c37f41cc3ea6d9a33ce34197c5a371304fc2665deff11/inputs.npz", "manifest": "artifacts/b3e7f630ac8ff0f5353c37f41cc3ea6d9a33ce34197c5a371304fc2665deff11/manifest.json", "model": "artifacts/b3e7f630ac8ff0f5353c37f41cc3ea6d9a33ce34197c5a371304fc2665deff11/model.onnx", "outputs": "observations/81748235eb33a34124aad679391361848fb6eed683ceffae2bcbfb77f06341b9/outputs.npz" }, "candidate_group": "a4c73789756cf3e2fb2342a6380f0629bb720f3b128c89e26ed3afd2e4951bc6", "case_id": "b3e7f630ac8ff0f5353c37f41cc3ea6d9a33ce34197c5a371304fc2665deff11", "category": "softmax", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "backend": "reference", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 2.220446049250313e-16, "max_budget_ratio": 0.00020185873175002848, "max_rel": 2.220446049250313e-16, "max_ulp": 1, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 8, 17 ], "backend": "reference", "elements": 136, "expected_dtype": "float64", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 8.881784197001252e-16, "max_budget_ratio": 0.0002100763289896815, "max_rel": 2.2050742937458016e-16, "max_ulp": 1, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "backend": "ort", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 4.440892098500626e-16, "max_budget_ratio": 0.00040371746350005696, "max_rel": 4.440892098500626e-16, "max_ulp": 2, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 8.881784197001252e-16, "max_budget_ratio": 0.00029814837700225334, "max_rel": 3.0850441453380463e-16, "max_ulp": 2, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 8, 17 ], "backend": "ort", "elements": 136, "expected_dtype": "float64", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 8.881784197001252e-16, "max_budget_ratio": 0.00018737145188598515, "max_rel": 1.9140945445281596e-16, "max_ulp": 1, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } } ], "comparison": { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 8.881784197001252e-16, "max_budget_ratio": 0.0002100763289896815, "max_rel": 2.2050742937458016e-16, "max_ulp": 1, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, "comparison_key": "7f728cb7518807e771df3c59f5a88b7bf27d08a337b65d49401757c6e017a4e1", "contract": "log_probability", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "float64", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "LogSoftmax/ordinary/float64/4x17/seed-20260906", "observation_id": "81748235eb33a34124aad679391361848fb6eed683ceffae2bcbfb77f06341b9", "operation": "LogSoftmax", "operators": [ "LogSoftmax" ], "opset": 19, "optimization": "disabled", "output_hash": "a82d381d0eec66bfcbfa30052aa3b2e1348a98e6274715ed18c09d8024993436", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.003375, "p95_ms": 0.003709, "repeats": 7, "samples_ms": [ 0.003375, 0.003709, 0.003709, 0.003666, 0.003125, 0.003, 0.002833 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.00875, "p95_ms": 0.010179399999999998, "repeats": 7, "samples_ms": [ 0.010417, 0.009625, 0.008958, 0.00875, 0.008459, 0.008542, 0.008417 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-13, "rtol": 1e-12 }, "variant": "ordinary", "warnings": [] }, { "artifacts": { "bundle": "observations/3d1c6c9c264c2d34bce826b3426b67510f559e241a51e936819e69a49e232079/reproducer.zip", "bundle_sha256": "a61a97b1a4945697299a22ed12a8b1b02e6b82926511a747bd1c7a495a515175", "case": "artifacts/7fd92f5ab9dbbb6f185776841f2dba00d077fa88456e18ebfb53f7610cf5c6b2/case.json", "inputs": "artifacts/7fd92f5ab9dbbb6f185776841f2dba00d077fa88456e18ebfb53f7610cf5c6b2/inputs.npz", "manifest": "artifacts/7fd92f5ab9dbbb6f185776841f2dba00d077fa88456e18ebfb53f7610cf5c6b2/manifest.json", "model": "artifacts/7fd92f5ab9dbbb6f185776841f2dba00d077fa88456e18ebfb53f7610cf5c6b2/model.onnx", "outputs": "observations/3d1c6c9c264c2d34bce826b3426b67510f559e241a51e936819e69a49e232079/outputs.npz" }, "candidate_group": "422186517ab6dbc21052a24009db9b4318d41517dc4da8914ea5b966d13f911e", "case_id": "7fd92f5ab9dbbb6f185776841f2dba00d077fa88456e18ebfb53f7610cf5c6b2", "category": "softmax", "checks": [ { "backend": "reference", "name": "finite_output", "passed": false }, { "backend": "reference", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 6.65400960997431e-08, "max_budget_ratio": 0.00604909964543119, "max_rel": 6.65400960997431e-08, "max_ulp": 318527719, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 105 ], "backend": "reference", "elements": 420, "expected_dtype": "float32", "expected_shape": [ 4, 105 ], "first_mismatch": [ 0, 1 ], "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 141, "name": "batch_partition", "nonfinite": 141, "numerical_divergence": false, "passed": false, "reason": "nonfinite_output", "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 105 ], "backend": "reference", "elements": 420, "expected_dtype": "float32", "expected_shape": [ 4, 105 ], "first_mismatch": [ 0, 0 ], "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 141, "name": "batch_permutation", "nonfinite": 141, "numerical_divergence": false, "passed": false, "reason": "nonfinite_output", "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 105 ], "backend": "reference", "elements": 420, "expected_dtype": "float32", "expected_shape": [ 4, 105 ], "first_mismatch": [ 0, 1 ], "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 141, "name": "batch_reshape", "nonfinite": 141, "numerical_divergence": false, "passed": false, "reason": "nonfinite_output", "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 105 ], "backend": "reference", "elements": 840, "expected_dtype": "float32", "expected_shape": [ 8, 105 ], "first_mismatch": [ 0, 1 ], "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 282, "name": "batch_duplicate", "nonfinite": 282, "numerical_divergence": false, "passed": false, "reason": "nonfinite_output", "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 105 ], "backend": "reference", "elements": 420, "expected_dtype": "float32", "expected_shape": [ 4, 105 ], "first_mismatch": [ 0, 1 ], "max_abs": 0.413299560546875, "max_budget_ratio": 399.7909031289195, "max_rel": 0.004001780013497845, "max_ulp": 54172, "mismatched": 166, "name": "equivalent_graph", "nonfinite": 141, "numerical_divergence": true, "passed": false, "reason": "nonfinite_output", "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "backend": "ort", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 4.710487688797116e-08, "max_budget_ratio": 0.004282261535270105, "max_rel": 4.710487688797116e-08, "max_ulp": 318527527, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 105 ], "backend": "ort", "elements": 420, "expected_dtype": "float32", "expected_shape": [ 4, 105 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 105 ], "backend": "ort", "elements": 420, "expected_dtype": "float32", "expected_shape": [ 4, 105 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 105 ], "backend": "ort", "elements": 420, "expected_dtype": "float32", "expected_shape": [ 4, 105 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 105 ], "backend": "ort", "elements": 840, "expected_dtype": "float32", "expected_shape": [ 8, 105 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 105 ], "backend": "ort", "elements": 420, "expected_dtype": "float32", "expected_shape": [ 4, 105 ], "first_mismatch": null, "max_abs": 1.1827796697616577e-07, "max_budget_ratio": 0.10900839599317617, "max_rel": 1.3909264501092754e-05, "max_ulp": 127, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } } ], "comparison": { "actual_dtype": "float32", "actual_shape": [ 4, 105 ], "elements": 420, "expected_dtype": "float32", "expected_shape": [ 4, 105 ], "first_mismatch": [ 0, 1 ], "max_abs": 0.413299560546875, "max_budget_ratio": 399.7909031289195, "max_rel": 0.004001780013497845, "max_ulp": 54172, "mismatched": 166, "nonfinite": 141, "numerical_divergence": true, "passed": false, "reason": "nonfinite_output", "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, "comparison_key": "da77597736067858c504d8411b5534e3309b1a066c7c7ded500d069603093edb", "contract": "log_probability", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "float32", "shape": [ 4, 105 ] } }, "ir_version": 10, "name": "LogSoftmax/random-001/float32/4x105/seed-20260906", "observation_id": "3d1c6c9c264c2d34bce826b3426b67510f559e241a51e936819e69a49e232079", "operation": "LogSoftmax", "operators": [ "LogSoftmax" ], "opset": 19, "optimization": "all", "output_hash": "afe8bb6529aff2c032285b9276da532d7f67c95d32c77c84c5d2f746b647af68", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "divergence", "timings": { "ort": { "median_ms": 0.003125, "p95_ms": 0.0034785999999999997, "repeats": 7, "samples_ms": [ 0.00325, 0.003541, 0.003333, 0.003125, 0.003125, 0.002916, 0.002834 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.012584, "p95_ms": 0.014270399999999999, "repeats": 7, "samples_ms": [ 0.014583, 0.013541, 0.012541, 0.012958, 0.012584, 0.012416, 0.01225 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-06, "rtol": 1e-05 }, "variant": "random-001", "warnings": [ { "backend": "reference", "message": "divide by zero encountered in log" }, { "backend": "reference", "message": "underflow encountered in divide" }, { "backend": "reference", "message": "underflow encountered in exp" } ] }, { "artifacts": { "bundle": "observations/d3324e1abed14a5da519d1cb8ecd4a9c069c763d106fbdb7ea8b9f5b769aca5b/reproducer.zip", "bundle_sha256": "c35eec0492f2a1489ec5cffb6d55a0cc33a48a17b591f245517ecc3a1428bdff", "case": "artifacts/7fd92f5ab9dbbb6f185776841f2dba00d077fa88456e18ebfb53f7610cf5c6b2/case.json", "inputs": "artifacts/7fd92f5ab9dbbb6f185776841f2dba00d077fa88456e18ebfb53f7610cf5c6b2/inputs.npz", "manifest": "artifacts/7fd92f5ab9dbbb6f185776841f2dba00d077fa88456e18ebfb53f7610cf5c6b2/manifest.json", "model": "artifacts/7fd92f5ab9dbbb6f185776841f2dba00d077fa88456e18ebfb53f7610cf5c6b2/model.onnx", "outputs": "observations/d3324e1abed14a5da519d1cb8ecd4a9c069c763d106fbdb7ea8b9f5b769aca5b/outputs.npz" }, "candidate_group": "422186517ab6dbc21052a24009db9b4318d41517dc4da8914ea5b966d13f911e", "case_id": "7fd92f5ab9dbbb6f185776841f2dba00d077fa88456e18ebfb53f7610cf5c6b2", "category": "softmax", "checks": [ { "backend": "reference", "name": "finite_output", "passed": false }, { "backend": "reference", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 6.65400960997431e-08, "max_budget_ratio": 0.00604909964543119, "max_rel": 6.65400960997431e-08, "max_ulp": 318527719, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 105 ], "backend": "reference", "elements": 420, "expected_dtype": "float32", "expected_shape": [ 4, 105 ], "first_mismatch": [ 0, 1 ], "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 141, "name": "batch_partition", "nonfinite": 141, "numerical_divergence": false, "passed": false, "reason": "nonfinite_output", "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 105 ], "backend": "reference", "elements": 420, "expected_dtype": "float32", "expected_shape": [ 4, 105 ], "first_mismatch": [ 0, 0 ], "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 141, "name": "batch_permutation", "nonfinite": 141, "numerical_divergence": false, "passed": false, "reason": "nonfinite_output", "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 105 ], "backend": "reference", "elements": 420, "expected_dtype": "float32", "expected_shape": [ 4, 105 ], "first_mismatch": [ 0, 1 ], "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 141, "name": "batch_reshape", "nonfinite": 141, "numerical_divergence": false, "passed": false, "reason": "nonfinite_output", "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 105 ], "backend": "reference", "elements": 840, "expected_dtype": "float32", "expected_shape": [ 8, 105 ], "first_mismatch": [ 0, 1 ], "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 282, "name": "batch_duplicate", "nonfinite": 282, "numerical_divergence": false, "passed": false, "reason": "nonfinite_output", "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 105 ], "backend": "reference", "elements": 420, "expected_dtype": "float32", "expected_shape": [ 4, 105 ], "first_mismatch": [ 0, 1 ], "max_abs": 0.413299560546875, "max_budget_ratio": 399.7909031289195, "max_rel": 0.004001780013497845, "max_ulp": 54172, "mismatched": 166, "name": "equivalent_graph", "nonfinite": 141, "numerical_divergence": true, "passed": false, "reason": "nonfinite_output", "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "backend": "ort", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 4.710487688797116e-08, "max_budget_ratio": 0.004282261535270105, "max_rel": 4.710487688797116e-08, "max_ulp": 318527527, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 105 ], "backend": "ort", "elements": 420, "expected_dtype": "float32", "expected_shape": [ 4, 105 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 105 ], "backend": "ort", "elements": 420, "expected_dtype": "float32", "expected_shape": [ 4, 105 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 105 ], "backend": "ort", "elements": 420, "expected_dtype": "float32", "expected_shape": [ 4, 105 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 105 ], "backend": "ort", "elements": 840, "expected_dtype": "float32", "expected_shape": [ 8, 105 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 105 ], "backend": "ort", "elements": 420, "expected_dtype": "float32", "expected_shape": [ 4, 105 ], "first_mismatch": null, "max_abs": 1.1827796697616577e-07, "max_budget_ratio": 0.10900839599317617, "max_rel": 1.3909264501092754e-05, "max_ulp": 127, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } } ], "comparison": { "actual_dtype": "float32", "actual_shape": [ 4, 105 ], "elements": 420, "expected_dtype": "float32", "expected_shape": [ 4, 105 ], "first_mismatch": [ 0, 1 ], "max_abs": 0.413299560546875, "max_budget_ratio": 399.7909031289195, "max_rel": 0.004001780013497845, "max_ulp": 54172, "mismatched": 166, "nonfinite": 141, "numerical_divergence": true, "passed": false, "reason": "nonfinite_output", "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, "comparison_key": "e06e5acd2825508e7a3d2dfa2811f850ee656d0e959857d15a2c47a29255356d", "contract": "log_probability", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "float32", "shape": [ 4, 105 ] } }, "ir_version": 10, "name": "LogSoftmax/random-001/float32/4x105/seed-20260906", "observation_id": "d3324e1abed14a5da519d1cb8ecd4a9c069c763d106fbdb7ea8b9f5b769aca5b", "operation": "LogSoftmax", "operators": [ "LogSoftmax" ], "opset": 19, "optimization": "disabled", "output_hash": "afe8bb6529aff2c032285b9276da532d7f67c95d32c77c84c5d2f746b647af68", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "divergence", "timings": { "ort": { "median_ms": 0.003167, "p95_ms": 0.0038205999999999995, "repeats": 7, "samples_ms": [ 0.00325, 0.003958, 0.0035, 0.003167, 0.003083, 0.002959, 0.003 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.012958, "p95_ms": 0.0144832, "repeats": 7, "samples_ms": [ 0.014833, 0.013667, 0.013083, 0.012958, 0.012875, 0.012875, 0.012708 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-06, "rtol": 1e-05 }, "variant": "random-001", "warnings": [ { "backend": "reference", "message": "divide by zero encountered in log" }, { "backend": "reference", "message": "underflow encountered in divide" }, { "backend": "reference", "message": "underflow encountered in exp" } ] }, { "artifacts": { "bundle": "observations/ac1945c089b71366de04231a5ee08652ab97e970a0d233d10f7b0476f158ede0/reproducer.zip", "bundle_sha256": "012f02cfdad1a2b903434d5a3903d882c8caf8c76fb421076643e3abb4cab6d5", "case": "artifacts/69519cc6a7202843566eff6a39f7b045f43da9b27362b15608accd1cc75af42e/case.json", "inputs": "artifacts/69519cc6a7202843566eff6a39f7b045f43da9b27362b15608accd1cc75af42e/inputs.npz", "manifest": "artifacts/69519cc6a7202843566eff6a39f7b045f43da9b27362b15608accd1cc75af42e/manifest.json", "model": "artifacts/69519cc6a7202843566eff6a39f7b045f43da9b27362b15608accd1cc75af42e/model.onnx", "outputs": "observations/ac1945c089b71366de04231a5ee08652ab97e970a0d233d10f7b0476f158ede0/outputs.npz" }, "candidate_group": "103e248d29653e5cebf56c68baf6552078b6c9c90c7b68afa7773b99d04a88be", "case_id": "69519cc6a7202843566eff6a39f7b045f43da9b27362b15608accd1cc75af42e", "category": "softmax", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "backend": "reference", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 9.286291402510471e-08, "max_budget_ratio": 0.008442083093191336, "max_rel": 9.286291402510471e-08, "max_ulp": 836434770, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 39 ], "backend": "reference", "elements": 156, "expected_dtype": "float32", "expected_shape": [ 4, 39 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 39 ], "backend": "reference", "elements": 156, "expected_dtype": "float32", "expected_shape": [ 4, 39 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 39 ], "backend": "reference", "elements": 156, "expected_dtype": "float32", "expected_shape": [ 4, 39 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 39 ], "backend": "reference", "elements": 312, "expected_dtype": "float32", "expected_shape": [ 8, 39 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 39 ], "backend": "reference", "elements": 156, "expected_dtype": "float32", "expected_shape": [ 4, 39 ], "first_mismatch": null, "max_abs": 3.814697265625e-06, "max_budget_ratio": 0.015486293575569028, "max_rel": 9.185993307636436e-07, "max_ulp": 10, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "backend": "ort", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 7.279055536102419e-08, "max_budget_ratio": 0.006617323214638562, "max_rel": 7.279055536102419e-08, "max_ulp": 655639036, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 39 ], "backend": "ort", "elements": 156, "expected_dtype": "float32", "expected_shape": [ 4, 39 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 39 ], "backend": "ort", "elements": 156, "expected_dtype": "float32", "expected_shape": [ 4, 39 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 39 ], "backend": "ort", "elements": 156, "expected_dtype": "float32", "expected_shape": [ 4, 39 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 39 ], "backend": "ort", "elements": 312, "expected_dtype": "float32", "expected_shape": [ 8, 39 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 39 ], "backend": "ort", "elements": 156, "expected_dtype": "float32", "expected_shape": [ 4, 39 ], "first_mismatch": null, "max_abs": 7.450580596923828e-09, "max_budget_ratio": 0.0035895936073185894, "max_rel": 9.185948585143456e-08, "max_ulp": 1, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } } ], "comparison": { "actual_dtype": "float32", "actual_shape": [ 4, 39 ], "elements": 156, "expected_dtype": "float32", "expected_shape": [ 4, 39 ], "first_mismatch": null, "max_abs": 3.814697265625e-06, "max_budget_ratio": 0.08207735595051585, "max_rel": 4.8685764530473105e-06, "max_ulp": 53, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, "comparison_key": "6ea0e47d0fc2e71f5f24a234abc40327bb159116f5e4cbe03a4819dcbe19dfbc", "contract": "log_probability", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "float32", "shape": [ 4, 39 ] } }, "ir_version": 10, "name": "LogSoftmax/random-008/float32/4x39/seed-20260906", "observation_id": "ac1945c089b71366de04231a5ee08652ab97e970a0d233d10f7b0476f158ede0", "operation": "LogSoftmax", "operators": [ "LogSoftmax" ], "opset": 19, "optimization": "all", "output_hash": "d754b5c80d936a75183eff80ee84196aaf5770db2d4aeba73483b33232ed146c", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.003083, "p95_ms": 0.0038249999999999994, "repeats": 7, "samples_ms": [ 0.003083, 0.004125, 0.003125, 0.003125, 0.00275, 0.003083, 0.002791 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.009334, "p95_ms": 0.0104669, "repeats": 7, "samples_ms": [ 0.010667, 0.01, 0.009625, 0.009334, 0.00925, 0.009, 0.009042 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-06, "rtol": 1e-05 }, "variant": "random-008", "warnings": [] }, { "artifacts": { "bundle": "observations/fbe3199c8849a572056620d643b89066314c1f17b789b781423081f28027122e/reproducer.zip", "bundle_sha256": "1d43a9f7dca4a10edb1e49d6258bfc4558ffe8f2d7b215e74b6cdebbdc987f54", "case": "artifacts/69519cc6a7202843566eff6a39f7b045f43da9b27362b15608accd1cc75af42e/case.json", "inputs": "artifacts/69519cc6a7202843566eff6a39f7b045f43da9b27362b15608accd1cc75af42e/inputs.npz", "manifest": "artifacts/69519cc6a7202843566eff6a39f7b045f43da9b27362b15608accd1cc75af42e/manifest.json", "model": "artifacts/69519cc6a7202843566eff6a39f7b045f43da9b27362b15608accd1cc75af42e/model.onnx", "outputs": "observations/fbe3199c8849a572056620d643b89066314c1f17b789b781423081f28027122e/outputs.npz" }, "candidate_group": "103e248d29653e5cebf56c68baf6552078b6c9c90c7b68afa7773b99d04a88be", "case_id": "69519cc6a7202843566eff6a39f7b045f43da9b27362b15608accd1cc75af42e", "category": "softmax", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "backend": "reference", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 9.286291402510471e-08, "max_budget_ratio": 0.008442083093191336, "max_rel": 9.286291402510471e-08, "max_ulp": 836434770, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 39 ], "backend": "reference", "elements": 156, "expected_dtype": "float32", "expected_shape": [ 4, 39 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 39 ], "backend": "reference", "elements": 156, "expected_dtype": "float32", "expected_shape": [ 4, 39 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 39 ], "backend": "reference", "elements": 156, "expected_dtype": "float32", "expected_shape": [ 4, 39 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 39 ], "backend": "reference", "elements": 312, "expected_dtype": "float32", "expected_shape": [ 8, 39 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 39 ], "backend": "reference", "elements": 156, "expected_dtype": "float32", "expected_shape": [ 4, 39 ], "first_mismatch": null, "max_abs": 3.814697265625e-06, "max_budget_ratio": 0.015486293575569028, "max_rel": 9.185993307636436e-07, "max_ulp": 10, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "backend": "ort", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 7.279055536102419e-08, "max_budget_ratio": 0.006617323214638562, "max_rel": 7.279055536102419e-08, "max_ulp": 655639036, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 39 ], "backend": "ort", "elements": 156, "expected_dtype": "float32", "expected_shape": [ 4, 39 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 39 ], "backend": "ort", "elements": 156, "expected_dtype": "float32", "expected_shape": [ 4, 39 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 39 ], "backend": "ort", "elements": 156, "expected_dtype": "float32", "expected_shape": [ 4, 39 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 39 ], "backend": "ort", "elements": 312, "expected_dtype": "float32", "expected_shape": [ 8, 39 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 39 ], "backend": "ort", "elements": 156, "expected_dtype": "float32", "expected_shape": [ 4, 39 ], "first_mismatch": null, "max_abs": 7.450580596923828e-09, "max_budget_ratio": 0.0035895936073185894, "max_rel": 9.185948585143456e-08, "max_ulp": 1, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } } ], "comparison": { "actual_dtype": "float32", "actual_shape": [ 4, 39 ], "elements": 156, "expected_dtype": "float32", "expected_shape": [ 4, 39 ], "first_mismatch": null, "max_abs": 3.814697265625e-06, "max_budget_ratio": 0.08207735595051585, "max_rel": 4.8685764530473105e-06, "max_ulp": 53, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, "comparison_key": "8c08b4f9d11e7e4ac08ab8a7277eaf4458a96ae3f8c20721d658609163b62e16", "contract": "log_probability", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "float32", "shape": [ 4, 39 ] } }, "ir_version": 10, "name": "LogSoftmax/random-008/float32/4x39/seed-20260906", "observation_id": "fbe3199c8849a572056620d643b89066314c1f17b789b781423081f28027122e", "operation": "LogSoftmax", "operators": [ "LogSoftmax" ], "opset": 19, "optimization": "disabled", "output_hash": "d754b5c80d936a75183eff80ee84196aaf5770db2d4aeba73483b33232ed146c", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.003167, "p95_ms": 0.003292, "repeats": 7, "samples_ms": [ 0.003292, 0.003292, 0.003208, 0.003167, 0.003, 0.002916, 0.002916 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.00925, "p95_ms": 0.010717099999999999, "repeats": 7, "samples_ms": [ 0.011042, 0.009959, 0.009417, 0.009, 0.009166, 0.00925, 0.008875 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-06, "rtol": 1e-05 }, "variant": "random-008", "warnings": [] }, { "artifacts": { "bundle": "observations/c04c60b76d8d1938e64719d11a38f799fb8cec9c84c06c5bf47d51f64afd79a3/reproducer.zip", "bundle_sha256": "ad482f08497eb72b71d64bf7e9cbb4f2c914ae6995e755fb2efa16ff03166464", "case": "artifacts/ce343b58f2631c4ae21355b2fb7de6119e34a17084bc7dae563aba24080f772a/case.json", "inputs": "artifacts/ce343b58f2631c4ae21355b2fb7de6119e34a17084bc7dae563aba24080f772a/inputs.npz", "manifest": "artifacts/ce343b58f2631c4ae21355b2fb7de6119e34a17084bc7dae563aba24080f772a/manifest.json", "model": "artifacts/ce343b58f2631c4ae21355b2fb7de6119e34a17084bc7dae563aba24080f772a/model.onnx", "outputs": "observations/c04c60b76d8d1938e64719d11a38f799fb8cec9c84c06c5bf47d51f64afd79a3/outputs.npz" }, "candidate_group": "884fbc15ab4745c9aaaabe07f0bb4483291fbd5a95da4ba2402586183500b6f0", "case_id": "ce343b58f2631c4ae21355b2fb7de6119e34a17084bc7dae563aba24080f772a", "category": "softmax", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "backend": "reference", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 8.29010593506041e-08, "max_budget_ratio": 0.007536459940964008, "max_rel": 8.29010593506041e-08, "max_ulp": 373353180, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 8 ], "backend": "reference", "elements": 32, "expected_dtype": "float32", "expected_shape": [ 4, 8 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 8 ], "backend": "reference", "elements": 32, "expected_dtype": "float32", "expected_shape": [ 4, 8 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 8 ], "backend": "reference", "elements": 32, "expected_dtype": "float32", "expected_shape": [ 4, 8 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 8 ], "backend": "reference", "elements": 64, "expected_dtype": "float32", "expected_shape": [ 8, 8 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 8 ], "backend": "reference", "elements": 32, "expected_dtype": "float32", "expected_shape": [ 4, 8 ], "first_mismatch": null, "max_abs": 2.384185791015625e-07, "max_budget_ratio": 0.011773082333478414, "max_rel": 3.199750982579532e-07, "max_ulp": 5, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "backend": "ort", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 7.659677536686615e-08, "max_budget_ratio": 0.00696334321516965, "max_rel": 7.659677536686615e-08, "max_ulp": 689922418, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 8 ], "backend": "ort", "elements": 32, "expected_dtype": "float32", "expected_shape": [ 4, 8 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 8 ], "backend": "ort", "elements": 32, "expected_dtype": "float32", "expected_shape": [ 4, 8 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 8 ], "backend": "ort", "elements": 32, "expected_dtype": "float32", "expected_shape": [ 4, 8 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 8 ], "backend": "ort", "elements": 64, "expected_dtype": "float32", "expected_shape": [ 8, 8 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 8 ], "backend": "ort", "elements": 32, "expected_dtype": "float32", "expected_shape": [ 4, 8 ], "first_mismatch": null, "max_abs": 1.4901161193847656e-08, "max_budget_ratio": 0.004787552501377929, "max_rel": 7.053871048042293e-08, "max_ulp": 1, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } } ], "comparison": { "actual_dtype": "float32", "actual_shape": [ 4, 8 ], "elements": 32, "expected_dtype": "float32", "expected_shape": [ 4, 8 ], "first_mismatch": null, "max_abs": 1.1920928955078125e-07, "max_budget_ratio": 0.05886541166739207, "max_rel": 1.5998754912897658e-06, "max_ulp": 25, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, "comparison_key": "1b50f85d3039f29beeab4b0490da9e8911db8ed3715bfcbb81977f55a30f5928", "contract": "log_probability", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "float32", "shape": [ 4, 8 ] } }, "ir_version": 10, "name": "LogSoftmax/random-015/float32/4x8/seed-20260906", "observation_id": "c04c60b76d8d1938e64719d11a38f799fb8cec9c84c06c5bf47d51f64afd79a3", "operation": "LogSoftmax", "operators": [ "LogSoftmax" ], "opset": 19, "optimization": "all", "output_hash": "32b70ddb35fe12cb008df6a40a6db6298703ab96dd16c121ec19e016ee36b7ba", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.002917, "p95_ms": 0.0035456999999999997, "repeats": 7, "samples_ms": [ 0.002917, 0.003708, 0.003167, 0.002917, 0.002917, 0.002958, 0.002667 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.0085, "p95_ms": 0.0092293, "repeats": 7, "samples_ms": [ 0.009292, 0.009083, 0.008833, 0.0085, 0.00825, 0.008209, 0.00825 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-06, "rtol": 1e-05 }, "variant": "random-015", "warnings": [] }, { "artifacts": { "bundle": "observations/4eb8ee7680b16c8ac977198c1e32500a29c9e385a90815d62729056b64ce6edb/reproducer.zip", "bundle_sha256": "e7d3aceb1fa37b9ff0deb13195df1f355e6c36c7d80b8a800347209b4eeb27df", "case": "artifacts/ce343b58f2631c4ae21355b2fb7de6119e34a17084bc7dae563aba24080f772a/case.json", "inputs": "artifacts/ce343b58f2631c4ae21355b2fb7de6119e34a17084bc7dae563aba24080f772a/inputs.npz", "manifest": "artifacts/ce343b58f2631c4ae21355b2fb7de6119e34a17084bc7dae563aba24080f772a/manifest.json", "model": "artifacts/ce343b58f2631c4ae21355b2fb7de6119e34a17084bc7dae563aba24080f772a/model.onnx", "outputs": "observations/4eb8ee7680b16c8ac977198c1e32500a29c9e385a90815d62729056b64ce6edb/outputs.npz" }, "candidate_group": "884fbc15ab4745c9aaaabe07f0bb4483291fbd5a95da4ba2402586183500b6f0", "case_id": "ce343b58f2631c4ae21355b2fb7de6119e34a17084bc7dae563aba24080f772a", "category": "softmax", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "backend": "reference", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 8.29010593506041e-08, "max_budget_ratio": 0.007536459940964008, "max_rel": 8.29010593506041e-08, "max_ulp": 373353180, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 8 ], "backend": "reference", "elements": 32, "expected_dtype": "float32", "expected_shape": [ 4, 8 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 8 ], "backend": "reference", "elements": 32, "expected_dtype": "float32", "expected_shape": [ 4, 8 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 8 ], "backend": "reference", "elements": 32, "expected_dtype": "float32", "expected_shape": [ 4, 8 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 8 ], "backend": "reference", "elements": 64, "expected_dtype": "float32", "expected_shape": [ 8, 8 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 8 ], "backend": "reference", "elements": 32, "expected_dtype": "float32", "expected_shape": [ 4, 8 ], "first_mismatch": null, "max_abs": 2.384185791015625e-07, "max_budget_ratio": 0.011773082333478414, "max_rel": 3.199750982579532e-07, "max_ulp": 5, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "backend": "ort", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 7.659677536686615e-08, "max_budget_ratio": 0.00696334321516965, "max_rel": 7.659677536686615e-08, "max_ulp": 689922418, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 8 ], "backend": "ort", "elements": 32, "expected_dtype": "float32", "expected_shape": [ 4, 8 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 8 ], "backend": "ort", "elements": 32, "expected_dtype": "float32", "expected_shape": [ 4, 8 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 8 ], "backend": "ort", "elements": 32, "expected_dtype": "float32", "expected_shape": [ 4, 8 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 8 ], "backend": "ort", "elements": 64, "expected_dtype": "float32", "expected_shape": [ 8, 8 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 8 ], "backend": "ort", "elements": 32, "expected_dtype": "float32", "expected_shape": [ 4, 8 ], "first_mismatch": null, "max_abs": 1.4901161193847656e-08, "max_budget_ratio": 0.004787552501377929, "max_rel": 7.053871048042293e-08, "max_ulp": 1, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } } ], "comparison": { "actual_dtype": "float32", "actual_shape": [ 4, 8 ], "elements": 32, "expected_dtype": "float32", "expected_shape": [ 4, 8 ], "first_mismatch": null, "max_abs": 1.1920928955078125e-07, "max_budget_ratio": 0.05886541166739207, "max_rel": 1.5998754912897658e-06, "max_ulp": 25, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, "comparison_key": "92f547d5807a18becf796b68befa3b51fc44cc1828ab1e27c4bb68e60ae39d5d", "contract": "log_probability", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "float32", "shape": [ 4, 8 ] } }, "ir_version": 10, "name": "LogSoftmax/random-015/float32/4x8/seed-20260906", "observation_id": "4eb8ee7680b16c8ac977198c1e32500a29c9e385a90815d62729056b64ce6edb", "operation": "LogSoftmax", "operators": [ "LogSoftmax" ], "opset": 19, "optimization": "disabled", "output_hash": "32b70ddb35fe12cb008df6a40a6db6298703ab96dd16c121ec19e016ee36b7ba", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.002667, "p95_ms": 0.0031794999999999996, "repeats": 7, "samples_ms": [ 0.002917, 0.003292, 0.002917, 0.002667, 0.002667, 0.002375, 0.0025 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.009167, "p95_ms": 0.06882489999999994, "repeats": 7, "samples_ms": [ 0.009584, 0.009084, 0.00875, 0.0085, 0.09325, 0.011833, 0.009167 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-06, "rtol": 1e-05 }, "variant": "random-015", "warnings": [] }, { "artifacts": { "bundle": "observations/d6c38b46eddd6457259d7fde79c0b5a940b45607687c3e380e607fe64cafe61f/reproducer.zip", "bundle_sha256": "345a4b55c1900801f1e8d12c45d7876c363d69911d5e50120bb73cb7717aaeba", "case": "artifacts/2fc18a24c42bbda7a98ab28226bfcb3cbbecb6096993cc486187e8ad12d6b31a/case.json", "inputs": "artifacts/2fc18a24c42bbda7a98ab28226bfcb3cbbecb6096993cc486187e8ad12d6b31a/inputs.npz", "manifest": "artifacts/2fc18a24c42bbda7a98ab28226bfcb3cbbecb6096993cc486187e8ad12d6b31a/manifest.json", "model": "artifacts/2fc18a24c42bbda7a98ab28226bfcb3cbbecb6096993cc486187e8ad12d6b31a/model.onnx", "outputs": "observations/d6c38b46eddd6457259d7fde79c0b5a940b45607687c3e380e607fe64cafe61f/outputs.npz" }, "candidate_group": "94e9d5c027c0e97df4d9acb212e36033193ce033c5c0f9b8c1ad97679e204405", "case_id": "2fc18a24c42bbda7a98ab28226bfcb3cbbecb6096993cc486187e8ad12d6b31a", "category": "softmax", "checks": [ { "backend": "reference", "name": "finite_output", "passed": false }, { "backend": "reference", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 1.4668779968474155e-08, "max_budget_ratio": 0.0013335254516794685, "max_rel": 1.4668779968474155e-08, "max_ulp": 66062312, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 127 ], "backend": "reference", "elements": 508, "expected_dtype": "float32", "expected_shape": [ 4, 127 ], "first_mismatch": [ 0, 0 ], "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 458, "name": "batch_partition", "nonfinite": 458, "numerical_divergence": false, "passed": false, "reason": "nonfinite_output", "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 127 ], "backend": "reference", "elements": 508, "expected_dtype": "float32", "expected_shape": [ 4, 127 ], "first_mismatch": [ 0, 0 ], "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 458, "name": "batch_permutation", "nonfinite": 458, "numerical_divergence": false, "passed": false, "reason": "nonfinite_output", "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 127 ], "backend": "reference", "elements": 508, "expected_dtype": "float32", "expected_shape": [ 4, 127 ], "first_mismatch": [ 0, 0 ], "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 458, "name": "batch_reshape", "nonfinite": 458, "numerical_divergence": false, "passed": false, "reason": "nonfinite_output", "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 127 ], "backend": "reference", "elements": 1016, "expected_dtype": "float32", "expected_shape": [ 8, 127 ], "first_mismatch": [ 0, 0 ], "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 916, "name": "batch_duplicate", "nonfinite": 916, "numerical_divergence": false, "passed": false, "reason": "nonfinite_output", "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 127 ], "backend": "reference", "elements": 508, "expected_dtype": "float32", "expected_shape": [ 4, 127 ], "first_mismatch": [ 0, 0 ], "max_abs": 0.2647857666015625, "max_budget_ratio": 256.1312686257159, "max_rel": 0.0025637926816151554, "max_ulp": 34706, "mismatched": 465, "name": "equivalent_graph", "nonfinite": 458, "numerical_divergence": true, "passed": false, "reason": "nonfinite_output", "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "backend": "ort", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 2.2133267441049043e-08, "max_budget_ratio": 0.0020121152219135493, "max_rel": 2.2133267441049043e-08, "max_ulp": 99679375, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 127 ], "backend": "ort", "elements": 508, "expected_dtype": "float32", "expected_shape": [ 4, 127 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 127 ], "backend": "ort", "elements": 508, "expected_dtype": "float32", "expected_shape": [ 4, 127 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 127 ], "backend": "ort", "elements": 508, "expected_dtype": "float32", "expected_shape": [ 4, 127 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 127 ], "backend": "ort", "elements": 1016, "expected_dtype": "float32", "expected_shape": [ 8, 127 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 127 ], "backend": "ort", "elements": 508, "expected_dtype": "float32", "expected_shape": [ 4, 127 ], "first_mismatch": null, "max_abs": 7.275957614183426e-12, "max_budget_ratio": 7.269674942836155e-06, "max_rel": 8.419005838327979e-08, "max_ulp": 1, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } } ], "comparison": { "actual_dtype": "float32", "actual_shape": [ 4, 127 ], "elements": 508, "expected_dtype": "float32", "expected_shape": [ 4, 127 ], "first_mismatch": [ 0, 0 ], "max_abs": 0.2647857666015625, "max_budget_ratio": 256.1312686257159, "max_rel": 0.0025637926816151554, "max_ulp": 34706, "mismatched": 465, "nonfinite": 458, "numerical_divergence": true, "passed": false, "reason": "nonfinite_output", "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, "comparison_key": "5cb80a23e689e2ad564075b0709d65efba92ccc4e39adb56525e7c67157c2289", "contract": "log_probability", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "float32", "shape": [ 4, 127 ] } }, "ir_version": 10, "name": "LogSoftmax/random-022/float32/4x127/seed-20260906", "observation_id": "d6c38b46eddd6457259d7fde79c0b5a940b45607687c3e380e607fe64cafe61f", "operation": "LogSoftmax", "operators": [ "LogSoftmax" ], "opset": 19, "optimization": "all", "output_hash": "51ae1b526fb21ae30a51c05ad640348c9cea7dc9df91cfe85f08f87be05169aa", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "divergence", "timings": { "ort": { "median_ms": 0.003083, "p95_ms": 0.0035250999999999998, "repeats": 7, "samples_ms": [ 0.003041, 0.003625, 0.003083, 0.003292, 0.002833, 0.003084, 0.002875 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.013792, "p95_ms": 0.07135899999999995, "repeats": 7, "samples_ms": [ 0.014542, 0.095209, 0.015709, 0.013792, 0.013583, 0.012958, 0.013084 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-06, "rtol": 1e-05 }, "variant": "random-022", "warnings": [ { "backend": "reference", "message": "divide by zero encountered in log" }, { "backend": "reference", "message": "underflow encountered in divide" }, { "backend": "reference", "message": "underflow encountered in exp" } ] }, { "artifacts": { "bundle": "observations/63adf2e7d5718be4ca91c4542b391b105b4feb221bd4666dbe388cfdce6ac322/reproducer.zip", "bundle_sha256": "d95d6637cf20396f077d5cc7f163a0a78f4e8f3155aef87ae387bad8d5f4468a", "case": "artifacts/2fc18a24c42bbda7a98ab28226bfcb3cbbecb6096993cc486187e8ad12d6b31a/case.json", "inputs": "artifacts/2fc18a24c42bbda7a98ab28226bfcb3cbbecb6096993cc486187e8ad12d6b31a/inputs.npz", "manifest": "artifacts/2fc18a24c42bbda7a98ab28226bfcb3cbbecb6096993cc486187e8ad12d6b31a/manifest.json", "model": "artifacts/2fc18a24c42bbda7a98ab28226bfcb3cbbecb6096993cc486187e8ad12d6b31a/model.onnx", "outputs": "observations/63adf2e7d5718be4ca91c4542b391b105b4feb221bd4666dbe388cfdce6ac322/outputs.npz" }, "candidate_group": "94e9d5c027c0e97df4d9acb212e36033193ce033c5c0f9b8c1ad97679e204405", "case_id": "2fc18a24c42bbda7a98ab28226bfcb3cbbecb6096993cc486187e8ad12d6b31a", "category": "softmax", "checks": [ { "backend": "reference", "name": "finite_output", "passed": false }, { "backend": "reference", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 1.4668779968474155e-08, "max_budget_ratio": 0.0013335254516794685, "max_rel": 1.4668779968474155e-08, "max_ulp": 66062312, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 127 ], "backend": "reference", "elements": 508, "expected_dtype": "float32", "expected_shape": [ 4, 127 ], "first_mismatch": [ 0, 0 ], "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 458, "name": "batch_partition", "nonfinite": 458, "numerical_divergence": false, "passed": false, "reason": "nonfinite_output", "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 127 ], "backend": "reference", "elements": 508, "expected_dtype": "float32", "expected_shape": [ 4, 127 ], "first_mismatch": [ 0, 0 ], "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 458, "name": "batch_permutation", "nonfinite": 458, "numerical_divergence": false, "passed": false, "reason": "nonfinite_output", "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 127 ], "backend": "reference", "elements": 508, "expected_dtype": "float32", "expected_shape": [ 4, 127 ], "first_mismatch": [ 0, 0 ], "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 458, "name": "batch_reshape", "nonfinite": 458, "numerical_divergence": false, "passed": false, "reason": "nonfinite_output", "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 127 ], "backend": "reference", "elements": 1016, "expected_dtype": "float32", "expected_shape": [ 8, 127 ], "first_mismatch": [ 0, 0 ], "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 916, "name": "batch_duplicate", "nonfinite": 916, "numerical_divergence": false, "passed": false, "reason": "nonfinite_output", "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 127 ], "backend": "reference", "elements": 508, "expected_dtype": "float32", "expected_shape": [ 4, 127 ], "first_mismatch": [ 0, 0 ], "max_abs": 0.2647857666015625, "max_budget_ratio": 256.1312686257159, "max_rel": 0.0025637926816151554, "max_ulp": 34706, "mismatched": 465, "name": "equivalent_graph", "nonfinite": 458, "numerical_divergence": true, "passed": false, "reason": "nonfinite_output", "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "backend": "ort", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 2.2133267441049043e-08, "max_budget_ratio": 0.0020121152219135493, "max_rel": 2.2133267441049043e-08, "max_ulp": 99679375, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 127 ], "backend": "ort", "elements": 508, "expected_dtype": "float32", "expected_shape": [ 4, 127 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 127 ], "backend": "ort", "elements": 508, "expected_dtype": "float32", "expected_shape": [ 4, 127 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 127 ], "backend": "ort", "elements": 508, "expected_dtype": "float32", "expected_shape": [ 4, 127 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 127 ], "backend": "ort", "elements": 1016, "expected_dtype": "float32", "expected_shape": [ 8, 127 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 127 ], "backend": "ort", "elements": 508, "expected_dtype": "float32", "expected_shape": [ 4, 127 ], "first_mismatch": null, "max_abs": 7.275957614183426e-12, "max_budget_ratio": 7.269674942836155e-06, "max_rel": 8.419005838327979e-08, "max_ulp": 1, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } } ], "comparison": { "actual_dtype": "float32", "actual_shape": [ 4, 127 ], "elements": 508, "expected_dtype": "float32", "expected_shape": [ 4, 127 ], "first_mismatch": [ 0, 0 ], "max_abs": 0.2647857666015625, "max_budget_ratio": 256.1312686257159, "max_rel": 0.0025637926816151554, "max_ulp": 34706, "mismatched": 465, "nonfinite": 458, "numerical_divergence": true, "passed": false, "reason": "nonfinite_output", "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, "comparison_key": "d038f920fcb00167d526ef31f3a83af3e8da6fbf8887f335f027938189067855", "contract": "log_probability", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "float32", "shape": [ 4, 127 ] } }, "ir_version": 10, "name": "LogSoftmax/random-022/float32/4x127/seed-20260906", "observation_id": "63adf2e7d5718be4ca91c4542b391b105b4feb221bd4666dbe388cfdce6ac322", "operation": "LogSoftmax", "operators": [ "LogSoftmax" ], "opset": 19, "optimization": "disabled", "output_hash": "51ae1b526fb21ae30a51c05ad640348c9cea7dc9df91cfe85f08f87be05169aa", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "divergence", "timings": { "ort": { "median_ms": 0.003041, "p95_ms": 0.0038335999999999995, "repeats": 7, "samples_ms": [ 0.003334, 0.003959, 0.003541, 0.003041, 0.003, 0.002791, 0.003 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.012833, "p95_ms": 0.0143079, "repeats": 7, "samples_ms": [ 0.014583, 0.013666, 0.013, 0.012833, 0.012541, 0.012625, 0.012542 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-06, "rtol": 1e-05 }, "variant": "random-022", "warnings": [ { "backend": "reference", "message": "divide by zero encountered in log" }, { "backend": "reference", "message": "underflow encountered in divide" }, { "backend": "reference", "message": "underflow encountered in exp" } ] }, { "artifacts": { "bundle": "observations/45607fce55f7e62a40de2e47c5f13ab33b7c877ce009522b1309009c574f7a7f/reproducer.zip", "bundle_sha256": "a37a5d078572e7415100d5bb23607e9fdf951cb3d107c2615f2b614b6a9ea4aa", "case": "artifacts/ea347c4dd6793ddbb6b06e820a5f95c66221d628f810b60ba80c61ebf893e92c/case.json", "inputs": "artifacts/ea347c4dd6793ddbb6b06e820a5f95c66221d628f810b60ba80c61ebf893e92c/inputs.npz", "manifest": "artifacts/ea347c4dd6793ddbb6b06e820a5f95c66221d628f810b60ba80c61ebf893e92c/manifest.json", "model": "artifacts/ea347c4dd6793ddbb6b06e820a5f95c66221d628f810b60ba80c61ebf893e92c/model.onnx", "outputs": "observations/45607fce55f7e62a40de2e47c5f13ab33b7c877ce009522b1309009c574f7a7f/outputs.npz" }, "candidate_group": "033e80a63afcf3d1d16b079173ca9beda2e6c58077e0c064911e71b294540dbb", "case_id": "ea347c4dd6793ddbb6b06e820a5f95c66221d628f810b60ba80c61ebf893e92c", "category": "softmax", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "backend": "reference", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 0.0009529663247072051, "max_budget_ratio": 0.17326660449221912, "max_rel": 0.0009529663247072051, "max_ulp": 4291778784848, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 8, 17 ], "backend": "reference", "elements": 136, "expected_dtype": "float16", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.001953125, "max_budget_ratio": 0.1332267519317879, "max_rel": 0.000689655172413793, "max_ulp": 1, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "backend": "ort", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 0.0007707337758059429, "max_budget_ratio": 0.14013341378289873, "max_rel": 0.0007707337758059429, "max_ulp": 6942152691043, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 8, 17 ], "backend": "ort", "elements": 136, "expected_dtype": "float16", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } } ], "comparison": { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.001953125, "max_budget_ratio": 0.1332267519317879, "max_rel": 0.000689655172413793, "max_ulp": 1, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, "comparison_key": "9dbecc486c5bd26c02853b3f207ecf11cfe2d0703d36e33c3393527ec84bfa38", "contract": "log_probability", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "float16", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "LogSoftmax/tiny/float16/4x17/seed-20260906", "observation_id": "45607fce55f7e62a40de2e47c5f13ab33b7c877ce009522b1309009c574f7a7f", "operation": "LogSoftmax", "operators": [ "LogSoftmax" ], "opset": 19, "optimization": "all", "output_hash": "edd1c836bf3c209b75978bd7d3023e64979353f015e6c18d753bd60de8784250", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.004833, "p95_ms": 0.0057668, "repeats": 7, "samples_ms": [ 0.005708, 0.005792, 0.005458, 0.004833, 0.004833, 0.0045, 0.004542 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.009542, "p95_ms": 0.0105376, "repeats": 7, "samples_ms": [ 0.01075, 0.010042, 0.0095, 0.009375, 0.009292, 0.009625, 0.009542 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 0.0005, "rtol": 0.005 }, "variant": "tiny", "warnings": [] }, { "artifacts": { "bundle": "observations/9707c15bf57511e89b62029df853a006d93da5fb33c9ccb1aa1649214f1594ba/reproducer.zip", "bundle_sha256": "c7a002f59ebf54a60b40f6724a7a72fcdb05101dc1ffbde6dab3017c96b1500e", "case": "artifacts/ea347c4dd6793ddbb6b06e820a5f95c66221d628f810b60ba80c61ebf893e92c/case.json", "inputs": "artifacts/ea347c4dd6793ddbb6b06e820a5f95c66221d628f810b60ba80c61ebf893e92c/inputs.npz", "manifest": "artifacts/ea347c4dd6793ddbb6b06e820a5f95c66221d628f810b60ba80c61ebf893e92c/manifest.json", "model": "artifacts/ea347c4dd6793ddbb6b06e820a5f95c66221d628f810b60ba80c61ebf893e92c/model.onnx", "outputs": "observations/9707c15bf57511e89b62029df853a006d93da5fb33c9ccb1aa1649214f1594ba/outputs.npz" }, "candidate_group": "033e80a63afcf3d1d16b079173ca9beda2e6c58077e0c064911e71b294540dbb", "case_id": "ea347c4dd6793ddbb6b06e820a5f95c66221d628f810b60ba80c61ebf893e92c", "category": "softmax", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "backend": "reference", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 0.0009529663247072051, "max_budget_ratio": 0.17326660449221912, "max_rel": 0.0009529663247072051, "max_ulp": 4291778784848, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 8, 17 ], "backend": "reference", "elements": 136, "expected_dtype": "float16", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.001953125, "max_budget_ratio": 0.1332267519317879, "max_rel": 0.000689655172413793, "max_ulp": 1, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "backend": "ort", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 0.0007707337758059429, "max_budget_ratio": 0.14013341378289873, "max_rel": 0.0007707337758059429, "max_ulp": 6942152691043, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 8, 17 ], "backend": "ort", "elements": 136, "expected_dtype": "float16", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } } ], "comparison": { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.001953125, "max_budget_ratio": 0.1332267519317879, "max_rel": 0.000689655172413793, "max_ulp": 1, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, "comparison_key": "66b9b09a2b09e0184276416ffe82a04d153a0538f818daf65a4d8cbb43b7c3d7", "contract": "log_probability", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "float16", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "LogSoftmax/tiny/float16/4x17/seed-20260906", "observation_id": "9707c15bf57511e89b62029df853a006d93da5fb33c9ccb1aa1649214f1594ba", "operation": "LogSoftmax", "operators": [ "LogSoftmax" ], "opset": 19, "optimization": "disabled", "output_hash": "edd1c836bf3c209b75978bd7d3023e64979353f015e6c18d753bd60de8784250", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.005084, "p95_ms": 0.011537399999999995, "repeats": 7, "samples_ms": [ 0.013875, 0.006083, 0.005084, 0.005125, 0.004916, 0.004875, 0.004708 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.009625, "p95_ms": 0.010587699999999999, "repeats": 7, "samples_ms": [ 0.01075, 0.010209, 0.009709, 0.009375, 0.009333, 0.009625, 0.009334 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 0.0005, "rtol": 0.005 }, "variant": "tiny", "warnings": [] }, { "artifacts": { "bundle": "observations/1f5a584010e69f50b2d66184294f1d7a576d4931d3c6c6b70573ebae9b00bf17/reproducer.zip", "bundle_sha256": "963cb347124b108983e5d93633daddd6524b8171956fbe462a560dbfae9e6937", "case": "artifacts/d7f0bfa594b260d2e5c5287652924fe6f2765409acee126587acbea3418b01b7/case.json", "inputs": "artifacts/d7f0bfa594b260d2e5c5287652924fe6f2765409acee126587acbea3418b01b7/inputs.npz", "manifest": "artifacts/d7f0bfa594b260d2e5c5287652924fe6f2765409acee126587acbea3418b01b7/manifest.json", "model": "artifacts/d7f0bfa594b260d2e5c5287652924fe6f2765409acee126587acbea3418b01b7/model.onnx", "outputs": "observations/1f5a584010e69f50b2d66184294f1d7a576d4931d3c6c6b70573ebae9b00bf17/outputs.npz" }, "candidate_group": "9cb0b7475a96052a21dc0e57f8e931dca9d2c1f31fe75368042f437308cc51cc", "case_id": "d7f0bfa594b260d2e5c5287652924fe6f2765409acee126587acbea3418b01b7", "category": "softmax", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "backend": "reference", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 7.670854196284438e-08, "max_budget_ratio": 0.006973503814804034, "max_rel": 7.670854196284438e-08, "max_ulp": 444965546, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 17 ], "backend": "reference", "elements": 136, "expected_dtype": "float32", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 2.384185791015625e-07, "max_budget_ratio": 0.008128857081718847, "max_rel": 8.415792672571299e-08, "max_ulp": 1, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "backend": "ort", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 1.3280310007246499e-07, "max_budget_ratio": 0.012073009097496815, "max_rel": 1.3280310007246499e-07, "max_ulp": 971851677, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 17 ], "backend": "ort", "elements": 136, "expected_dtype": "float32", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 2.384185791015625e-07, "max_budget_ratio": 0.008128857742502075, "max_rel": 8.415793380827022e-08, "max_ulp": 1, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } } ], "comparison": { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 2.384185791015625e-07, "max_budget_ratio": 0.008128857081718847, "max_rel": 8.415792672571299e-08, "max_ulp": 1, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, "comparison_key": "cf4e79fe0b592353dcbda68612068440a13fbef79b3e6f443a6560c2a13636a2", "contract": "log_probability", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "float32", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "LogSoftmax/tiny/float32/4x17/seed-20260906", "observation_id": "1f5a584010e69f50b2d66184294f1d7a576d4931d3c6c6b70573ebae9b00bf17", "operation": "LogSoftmax", "operators": [ "LogSoftmax" ], "opset": 19, "optimization": "all", "output_hash": "616ad504ed1219290d4969226c31d31b0dc63c4e6cf6043964e14f1614505359", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.003084, "p95_ms": 0.0036041999999999992, "repeats": 7, "samples_ms": [ 0.003125, 0.003792, 0.003166, 0.003084, 0.003083, 0.002834, 0.00275 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.008792, "p95_ms": 0.0099043, "repeats": 7, "samples_ms": [ 0.010042, 0.009583, 0.008917, 0.00875, 0.008666, 0.008792, 0.008541 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-06, "rtol": 1e-05 }, "variant": "tiny", "warnings": [] }, { "artifacts": { "bundle": "observations/00e86822d67b24b8da835a66baee017433297f57e5808e50d3e2020a71644e86/reproducer.zip", "bundle_sha256": "adc894316364635c56e8e757bae9ece45f8d2d9d0585c44682b3fdaea2cd7f99", "case": "artifacts/d7f0bfa594b260d2e5c5287652924fe6f2765409acee126587acbea3418b01b7/case.json", "inputs": "artifacts/d7f0bfa594b260d2e5c5287652924fe6f2765409acee126587acbea3418b01b7/inputs.npz", "manifest": "artifacts/d7f0bfa594b260d2e5c5287652924fe6f2765409acee126587acbea3418b01b7/manifest.json", "model": "artifacts/d7f0bfa594b260d2e5c5287652924fe6f2765409acee126587acbea3418b01b7/model.onnx", "outputs": "observations/00e86822d67b24b8da835a66baee017433297f57e5808e50d3e2020a71644e86/outputs.npz" }, "candidate_group": "9cb0b7475a96052a21dc0e57f8e931dca9d2c1f31fe75368042f437308cc51cc", "case_id": "d7f0bfa594b260d2e5c5287652924fe6f2765409acee126587acbea3418b01b7", "category": "softmax", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "backend": "reference", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 7.670854196284438e-08, "max_budget_ratio": 0.006973503814804034, "max_rel": 7.670854196284438e-08, "max_ulp": 444965546, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 17 ], "backend": "reference", "elements": 136, "expected_dtype": "float32", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 2.384185791015625e-07, "max_budget_ratio": 0.008128857081718847, "max_rel": 8.415792672571299e-08, "max_ulp": 1, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "backend": "ort", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 1.3280310007246499e-07, "max_budget_ratio": 0.012073009097496815, "max_rel": 1.3280310007246499e-07, "max_ulp": 971851677, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 17 ], "backend": "ort", "elements": 136, "expected_dtype": "float32", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 2.384185791015625e-07, "max_budget_ratio": 0.008128857742502075, "max_rel": 8.415793380827022e-08, "max_ulp": 1, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } } ], "comparison": { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 2.384185791015625e-07, "max_budget_ratio": 0.008128857081718847, "max_rel": 8.415792672571299e-08, "max_ulp": 1, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, "comparison_key": "b37a8cb120328a8fe3c3e5e1db2ea9be5c526db65e2ff85c40d3e64e9dc9da65", "contract": "log_probability", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "float32", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "LogSoftmax/tiny/float32/4x17/seed-20260906", "observation_id": "00e86822d67b24b8da835a66baee017433297f57e5808e50d3e2020a71644e86", "operation": "LogSoftmax", "operators": [ "LogSoftmax" ], "opset": 19, "optimization": "disabled", "output_hash": "616ad504ed1219290d4969226c31d31b0dc63c4e6cf6043964e14f1614505359", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.003042, "p95_ms": 0.0037123999999999994, "repeats": 7, "samples_ms": [ 0.003333, 0.003875, 0.003042, 0.003208, 0.002791, 0.002792, 0.002625 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.008666, "p95_ms": 0.0097501, "repeats": 7, "samples_ms": [ 0.01, 0.009167, 0.008875, 0.0085, 0.008375, 0.008666, 0.008334 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-06, "rtol": 1e-05 }, "variant": "tiny", "warnings": [] }, { "artifacts": { "bundle": "observations/e8282a67b5dc7fb8c304b04b351dee82e17969a281d43ba4e9c1389f36e4005d/reproducer.zip", "bundle_sha256": "64c9698a7d09d96f207314065c098922fe0ae9b435a1336d60d0e5960d95e36e", "case": "artifacts/a0bc9e3c608fafd2ff861e8f91b649ebb56039935e1b94cd42ff97f3ee7b7350/case.json", "inputs": "artifacts/a0bc9e3c608fafd2ff861e8f91b649ebb56039935e1b94cd42ff97f3ee7b7350/inputs.npz", "manifest": "artifacts/a0bc9e3c608fafd2ff861e8f91b649ebb56039935e1b94cd42ff97f3ee7b7350/manifest.json", "model": "artifacts/a0bc9e3c608fafd2ff861e8f91b649ebb56039935e1b94cd42ff97f3ee7b7350/model.onnx", "outputs": "observations/e8282a67b5dc7fb8c304b04b351dee82e17969a281d43ba4e9c1389f36e4005d/outputs.npz" }, "candidate_group": "7a8fe0f11f565f1feac60b6bf260f17a541c6f71e14161f73c038b085451973e", "case_id": "a0bc9e3c608fafd2ff861e8f91b649ebb56039935e1b94cd42ff97f3ee7b7350", "category": "softmax", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "backend": "reference", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 2.220446049250313e-16, "max_budget_ratio": 0.00020185873175002848, "max_rel": 2.220446049250313e-16, "max_ulp": 2, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 8, 17 ], "backend": "reference", "elements": 136, "expected_dtype": "float64", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 4.440892098500626e-16, "max_budget_ratio": 0.0001514084417676435, "max_rel": 1.5675279509150392e-16, "max_ulp": 1, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "backend": "ort", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 2.220446049250313e-16, "max_budget_ratio": 0.00020185873175002848, "max_rel": 2.220446049250313e-16, "max_ulp": 2, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 8, 17 ], "backend": "ort", "elements": 136, "expected_dtype": "float64", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 4.440892098500626e-16, "max_budget_ratio": 0.00015141324187443986, "max_rel": 1.567579400490241e-16, "max_ulp": 1, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } } ], "comparison": { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 4.440892098500626e-16, "max_budget_ratio": 0.0001514084417676435, "max_rel": 1.5675279509150392e-16, "max_ulp": 1, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, "comparison_key": "6bb8197bf91df98ce714170472f08d717bb8bf9df9cbdf858b060afe42f0d74e", "contract": "log_probability", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "float64", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "LogSoftmax/tiny/float64/4x17/seed-20260906", "observation_id": "e8282a67b5dc7fb8c304b04b351dee82e17969a281d43ba4e9c1389f36e4005d", "operation": "LogSoftmax", "operators": [ "LogSoftmax" ], "opset": 19, "optimization": "all", "output_hash": "24bb38bc41650f5e57b664932cd39373e9944e41877e171073c7eb550f15d4a2", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.003666, "p95_ms": 0.0040668, "repeats": 7, "samples_ms": [ 0.003833, 0.004167, 0.003666, 0.003458, 0.003167, 0.003417, 0.003667 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.009166, "p95_ms": 0.048216899999999965, "repeats": 7, "samples_ms": [ 0.063792, 0.011875, 0.01, 0.009, 0.008834, 0.009166, 0.008667 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-13, "rtol": 1e-12 }, "variant": "tiny", "warnings": [] }, { "artifacts": { "bundle": "observations/96037ea9ad8fa5d4a26e13b0f888992da06d6fe3acc530cb0bd9f6ae470ff1b9/reproducer.zip", "bundle_sha256": "192ad9f921b569ab95e0098847522d623579bb8edd976902dc12501ad013004d", "case": "artifacts/a0bc9e3c608fafd2ff861e8f91b649ebb56039935e1b94cd42ff97f3ee7b7350/case.json", "inputs": "artifacts/a0bc9e3c608fafd2ff861e8f91b649ebb56039935e1b94cd42ff97f3ee7b7350/inputs.npz", "manifest": "artifacts/a0bc9e3c608fafd2ff861e8f91b649ebb56039935e1b94cd42ff97f3ee7b7350/manifest.json", "model": "artifacts/a0bc9e3c608fafd2ff861e8f91b649ebb56039935e1b94cd42ff97f3ee7b7350/model.onnx", "outputs": "observations/96037ea9ad8fa5d4a26e13b0f888992da06d6fe3acc530cb0bd9f6ae470ff1b9/outputs.npz" }, "candidate_group": "7a8fe0f11f565f1feac60b6bf260f17a541c6f71e14161f73c038b085451973e", "case_id": "a0bc9e3c608fafd2ff861e8f91b649ebb56039935e1b94cd42ff97f3ee7b7350", "category": "softmax", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "backend": "reference", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 2.220446049250313e-16, "max_budget_ratio": 0.00020185873175002848, "max_rel": 2.220446049250313e-16, "max_ulp": 2, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 8, 17 ], "backend": "reference", "elements": 136, "expected_dtype": "float64", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 4.440892098500626e-16, "max_budget_ratio": 0.0001514084417676435, "max_rel": 1.5675279509150392e-16, "max_ulp": 1, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "backend": "ort", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 2.220446049250313e-16, "max_budget_ratio": 0.00020185873175002848, "max_rel": 2.220446049250313e-16, "max_ulp": 2, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 8, 17 ], "backend": "ort", "elements": 136, "expected_dtype": "float64", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 4.440892098500626e-16, "max_budget_ratio": 0.00015141324187443986, "max_rel": 1.567579400490241e-16, "max_ulp": 1, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } } ], "comparison": { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 4.440892098500626e-16, "max_budget_ratio": 0.0001514084417676435, "max_rel": 1.5675279509150392e-16, "max_ulp": 1, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, "comparison_key": "ba7aebe7f6f02bb6246fb47fac73789cdae97574e9c58e561d8fcb03653fe0a2", "contract": "log_probability", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "float64", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "LogSoftmax/tiny/float64/4x17/seed-20260906", "observation_id": "96037ea9ad8fa5d4a26e13b0f888992da06d6fe3acc530cb0bd9f6ae470ff1b9", "operation": "LogSoftmax", "operators": [ "LogSoftmax" ], "opset": 19, "optimization": "disabled", "output_hash": "24bb38bc41650f5e57b664932cd39373e9944e41877e171073c7eb550f15d4a2", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.00325, "p95_ms": 0.0038214, "repeats": 7, "samples_ms": [ 0.003792, 0.003834, 0.003375, 0.00325, 0.003083, 0.002958, 0.002875 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.008625, "p95_ms": 0.009649999999999999, "repeats": 7, "samples_ms": [ 0.009875, 0.009125, 0.00875, 0.008625, 0.0085, 0.0085, 0.008417 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-13, "rtol": 1e-12 }, "variant": "tiny", "warnings": [] }, { "artifacts": { "bundle": "observations/2755c1d1acafe07f84cad162ba29544aedd33e50af0d482c87561d2b0ece1362/reproducer.zip", "bundle_sha256": "767ba25d6adb2cfae09541c8f746474efd7ac8dc98c1c69b7caa0cd7324bf68e", "case": "artifacts/e530394b616a1973a0a095ecfde3a23cc208df2f13a3e8e7122d2de9a20bcbaf/case.json", "inputs": "artifacts/e530394b616a1973a0a095ecfde3a23cc208df2f13a3e8e7122d2de9a20bcbaf/inputs.npz", "manifest": "artifacts/e530394b616a1973a0a095ecfde3a23cc208df2f13a3e8e7122d2de9a20bcbaf/manifest.json", "model": "artifacts/e530394b616a1973a0a095ecfde3a23cc208df2f13a3e8e7122d2de9a20bcbaf/model.onnx", "outputs": "observations/2755c1d1acafe07f84cad162ba29544aedd33e50af0d482c87561d2b0ece1362/outputs.npz" }, "candidate_group": "7485d964345a6abf228524489e3c098a2086ab6ddd86d6913fce614f7e5e0658", "case_id": "e530394b616a1973a0a095ecfde3a23cc208df2f13a3e8e7122d2de9a20bcbaf", "category": "softmax", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "backend": "reference", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 0.00037488094367088554, "max_budget_ratio": 0.06816017157652465, "max_rel": 0.00037488094367088554, "max_ulp": 3376627356449, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 257 ], "backend": "reference", "elements": 1028, "expected_dtype": "float16", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 257 ], "backend": "reference", "elements": 1028, "expected_dtype": "float16", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 257 ], "backend": "reference", "elements": 1028, "expected_dtype": "float16", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 8, 257 ], "backend": "reference", "elements": 2056, "expected_dtype": "float16", "expected_shape": [ 8, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 257 ], "backend": "reference", "elements": 1028, "expected_dtype": "float16", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0078125, "max_budget_ratio": 0.18964536317087044, "max_rel": 0.0009718172983479105, "max_ulp": 1, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "backend": "ort", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 0.00015256442460476194, "max_budget_ratio": 0.0277389862917749, "max_rel": 0.00015256442460476194, "max_ulp": 687089085800, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 257 ], "backend": "ort", "elements": 1028, "expected_dtype": "float16", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 257 ], "backend": "ort", "elements": 1028, "expected_dtype": "float16", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 257 ], "backend": "ort", "elements": 1028, "expected_dtype": "float16", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 8, 257 ], "backend": "ort", "elements": 2056, "expected_dtype": "float16", "expected_shape": [ 8, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 257 ], "backend": "ort", "elements": 1028, "expected_dtype": "float16", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } } ], "comparison": { "actual_dtype": "float16", "actual_shape": [ 4, 257 ], "elements": 1028, "expected_dtype": "float16", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0078125, "max_budget_ratio": 0.19105846388995032, "max_rel": 0.0009671179883945841, "max_ulp": 1, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, "comparison_key": "2c403f053eab9823ad802798b768ab447c8aee80bf27b2aa84b5a36f258ab7a5", "contract": "log_probability", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "float16", "shape": [ 4, 257 ] } }, "ir_version": 10, "name": "LogSoftmax/wide/float16/4x257/seed-20260906", "observation_id": "2755c1d1acafe07f84cad162ba29544aedd33e50af0d482c87561d2b0ece1362", "operation": "LogSoftmax", "operators": [ "LogSoftmax" ], "opset": 19, "optimization": "all", "output_hash": "62e98bad5368babc3e2581cc06f7c865bbbe447239716edb9014376918f93417", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.00575, "p95_ms": 0.0062751, "repeats": 7, "samples_ms": [ 0.006042, 0.006375, 0.005834, 0.00575, 0.005458, 0.0055, 0.005417 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.025625, "p95_ms": 0.0265286, "repeats": 7, "samples_ms": [ 0.026666, 0.026208, 0.025875, 0.025625, 0.025584, 0.023167, 0.022458 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 0.0005, "rtol": 0.005 }, "variant": "wide", "warnings": [] }, { "artifacts": { "bundle": "observations/1246c1f748521a83900b0d695d9f98d3f0966b40c0b88829c0223891ac3e43f5/reproducer.zip", "bundle_sha256": "5784b24b2311e3803e1e80781b1318b3ede2a0afc9a168c1c621b9ea05397d07", "case": "artifacts/e530394b616a1973a0a095ecfde3a23cc208df2f13a3e8e7122d2de9a20bcbaf/case.json", "inputs": "artifacts/e530394b616a1973a0a095ecfde3a23cc208df2f13a3e8e7122d2de9a20bcbaf/inputs.npz", "manifest": "artifacts/e530394b616a1973a0a095ecfde3a23cc208df2f13a3e8e7122d2de9a20bcbaf/manifest.json", "model": "artifacts/e530394b616a1973a0a095ecfde3a23cc208df2f13a3e8e7122d2de9a20bcbaf/model.onnx", "outputs": "observations/1246c1f748521a83900b0d695d9f98d3f0966b40c0b88829c0223891ac3e43f5/outputs.npz" }, "candidate_group": "7485d964345a6abf228524489e3c098a2086ab6ddd86d6913fce614f7e5e0658", "case_id": "e530394b616a1973a0a095ecfde3a23cc208df2f13a3e8e7122d2de9a20bcbaf", "category": "softmax", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "backend": "reference", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 0.00037488094367088554, "max_budget_ratio": 0.06816017157652465, "max_rel": 0.00037488094367088554, "max_ulp": 3376627356449, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 257 ], "backend": "reference", "elements": 1028, "expected_dtype": "float16", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 257 ], "backend": "reference", "elements": 1028, "expected_dtype": "float16", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 257 ], "backend": "reference", "elements": 1028, "expected_dtype": "float16", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 8, 257 ], "backend": "reference", "elements": 2056, "expected_dtype": "float16", "expected_shape": [ 8, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 257 ], "backend": "reference", "elements": 1028, "expected_dtype": "float16", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0078125, "max_budget_ratio": 0.18964536317087044, "max_rel": 0.0009718172983479105, "max_ulp": 1, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "backend": "ort", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 0.00015256442460476194, "max_budget_ratio": 0.0277389862917749, "max_rel": 0.00015256442460476194, "max_ulp": 687089085800, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 257 ], "backend": "ort", "elements": 1028, "expected_dtype": "float16", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 257 ], "backend": "ort", "elements": 1028, "expected_dtype": "float16", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 257 ], "backend": "ort", "elements": 1028, "expected_dtype": "float16", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 8, 257 ], "backend": "ort", "elements": 2056, "expected_dtype": "float16", "expected_shape": [ 8, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 257 ], "backend": "ort", "elements": 1028, "expected_dtype": "float16", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } } ], "comparison": { "actual_dtype": "float16", "actual_shape": [ 4, 257 ], "elements": 1028, "expected_dtype": "float16", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0078125, "max_budget_ratio": 0.19105846388995032, "max_rel": 0.0009671179883945841, "max_ulp": 1, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, "comparison_key": "d9f209b159c4536256aacf5cf593ef3b2cc892b0e8b98ba6cbc9f8ca99b8321d", "contract": "log_probability", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "float16", "shape": [ 4, 257 ] } }, "ir_version": 10, "name": "LogSoftmax/wide/float16/4x257/seed-20260906", "observation_id": "1246c1f748521a83900b0d695d9f98d3f0966b40c0b88829c0223891ac3e43f5", "operation": "LogSoftmax", "operators": [ "LogSoftmax" ], "opset": 19, "optimization": "disabled", "output_hash": "62e98bad5368babc3e2581cc06f7c865bbbe447239716edb9014376918f93417", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.006167, "p95_ms": 0.008820599999999998, "repeats": 7, "samples_ms": [ 0.006167, 0.007625, 0.006833, 0.009333, 0.005958, 0.005958, 0.005416 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.025167, "p95_ms": 0.0259456, "repeats": 7, "samples_ms": [ 0.026083, 0.025625, 0.025208, 0.025167, 0.025, 0.025167, 0.024875 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 0.0005, "rtol": 0.005 }, "variant": "wide", "warnings": [] }, { "artifacts": { "bundle": "observations/4173600c58594bfa6efcb39fa2044c04a663b1b857acbb45a5456c1387817f55/reproducer.zip", "bundle_sha256": "ca09b04773fa7de2003fe83badbb3d1c14498cad8e58837b71b3e047f3d16d25", "case": "artifacts/65ea16d0db4a6b982eed5e8184fb822b915809212c06c7ba3288642d5dfdc645/case.json", "inputs": "artifacts/65ea16d0db4a6b982eed5e8184fb822b915809212c06c7ba3288642d5dfdc645/inputs.npz", "manifest": "artifacts/65ea16d0db4a6b982eed5e8184fb822b915809212c06c7ba3288642d5dfdc645/manifest.json", "model": "artifacts/65ea16d0db4a6b982eed5e8184fb822b915809212c06c7ba3288642d5dfdc645/model.onnx", "outputs": "observations/4173600c58594bfa6efcb39fa2044c04a663b1b857acbb45a5456c1387817f55/outputs.npz" }, "candidate_group": "0df932d176d83318d5e8bf5253c1b7facf01be4e0a76d340863dd7f4276023fd", "case_id": "65ea16d0db4a6b982eed5e8184fb822b915809212c06c7ba3288642d5dfdc645", "category": "softmax", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "backend": "reference", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 4.0334860007540385e-08, "max_budget_ratio": 0.0036668054552309436, "max_rel": 4.0334860007540385e-08, "max_ulp": 363304121, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 257 ], "backend": "reference", "elements": 1028, "expected_dtype": "float32", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 257 ], "backend": "reference", "elements": 1028, "expected_dtype": "float32", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 257 ], "backend": "reference", "elements": 1028, "expected_dtype": "float32", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 257 ], "backend": "reference", "elements": 2056, "expected_dtype": "float32", "expected_shape": [ 8, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 257 ], "backend": "reference", "elements": 1028, "expected_dtype": "float32", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 9.5367431640625e-07, "max_budget_ratio": 0.011655980813136606, "max_rel": 1.1800205063963601e-07, "max_ulp": 1, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "backend": "ort", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 1.3035059698296436e-07, "max_budget_ratio": 0.011850054271178577, "max_rel": 1.3035059698296436e-07, "max_ulp": 587046900, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 257 ], "backend": "ort", "elements": 1028, "expected_dtype": "float32", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 257 ], "backend": "ort", "elements": 1028, "expected_dtype": "float32", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 257 ], "backend": "ort", "elements": 1028, "expected_dtype": "float32", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 257 ], "backend": "ort", "elements": 2056, "expected_dtype": "float32", "expected_shape": [ 8, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 257 ], "backend": "ort", "elements": 1028, "expected_dtype": "float32", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 9.5367431640625e-07, "max_budget_ratio": 0.011430180014133645, "max_rel": 1.1583994209856335e-07, "max_ulp": 1, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } } ], "comparison": { "actual_dtype": "float32", "actual_shape": [ 4, 257 ], "elements": 1028, "expected_dtype": "float32", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 9.5367431640625e-07, "max_budget_ratio": 0.011655980813136606, "max_rel": 1.1800205063963601e-07, "max_ulp": 1, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, "comparison_key": "0fd9a69383072eb28d31113ae640d25ae14c55b045f6d77f77c608f103df19a4", "contract": "log_probability", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "float32", "shape": [ 4, 257 ] } }, "ir_version": 10, "name": "LogSoftmax/wide/float32/4x257/seed-20260906", "observation_id": "4173600c58594bfa6efcb39fa2044c04a663b1b857acbb45a5456c1387817f55", "operation": "LogSoftmax", "operators": [ "LogSoftmax" ], "opset": 19, "optimization": "all", "output_hash": "c1f8294a9cf44681e166a42ab9640c9b5a11a2ed141caa08c6c284b101aed60e", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.0035, "p95_ms": 0.0039333, "repeats": 7, "samples_ms": [ 0.003209, 0.004083, 0.003542, 0.0035, 0.003, 0.003167, 0.003584 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.012084, "p95_ms": 0.0129375, "repeats": 7, "samples_ms": [ 0.013125, 0.0125, 0.012084, 0.012042, 0.011958, 0.012375, 0.011875 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-06, "rtol": 1e-05 }, "variant": "wide", "warnings": [] }, { "artifacts": { "bundle": "observations/8cbb4a9f6fb31d5ad3b25ce52a8a9afdcf1ee777c7968a5f3205785f6ec98e24/reproducer.zip", "bundle_sha256": "0d63d9157f41499a518674458984bacd4d2ecdb19db56c250b7e814528d2bec3", "case": "artifacts/65ea16d0db4a6b982eed5e8184fb822b915809212c06c7ba3288642d5dfdc645/case.json", "inputs": "artifacts/65ea16d0db4a6b982eed5e8184fb822b915809212c06c7ba3288642d5dfdc645/inputs.npz", "manifest": "artifacts/65ea16d0db4a6b982eed5e8184fb822b915809212c06c7ba3288642d5dfdc645/manifest.json", "model": "artifacts/65ea16d0db4a6b982eed5e8184fb822b915809212c06c7ba3288642d5dfdc645/model.onnx", "outputs": "observations/8cbb4a9f6fb31d5ad3b25ce52a8a9afdcf1ee777c7968a5f3205785f6ec98e24/outputs.npz" }, "candidate_group": "0df932d176d83318d5e8bf5253c1b7facf01be4e0a76d340863dd7f4276023fd", "case_id": "65ea16d0db4a6b982eed5e8184fb822b915809212c06c7ba3288642d5dfdc645", "category": "softmax", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "backend": "reference", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 4.0334860007540385e-08, "max_budget_ratio": 0.0036668054552309436, "max_rel": 4.0334860007540385e-08, "max_ulp": 363304121, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 257 ], "backend": "reference", "elements": 1028, "expected_dtype": "float32", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 257 ], "backend": "reference", "elements": 1028, "expected_dtype": "float32", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 257 ], "backend": "reference", "elements": 1028, "expected_dtype": "float32", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 257 ], "backend": "reference", "elements": 2056, "expected_dtype": "float32", "expected_shape": [ 8, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 257 ], "backend": "reference", "elements": 1028, "expected_dtype": "float32", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 9.5367431640625e-07, "max_budget_ratio": 0.011655980813136606, "max_rel": 1.1800205063963601e-07, "max_ulp": 1, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "backend": "ort", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 1.3035059698296436e-07, "max_budget_ratio": 0.011850054271178577, "max_rel": 1.3035059698296436e-07, "max_ulp": 587046900, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 257 ], "backend": "ort", "elements": 1028, "expected_dtype": "float32", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 257 ], "backend": "ort", "elements": 1028, "expected_dtype": "float32", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 257 ], "backend": "ort", "elements": 1028, "expected_dtype": "float32", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 257 ], "backend": "ort", "elements": 2056, "expected_dtype": "float32", "expected_shape": [ 8, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 257 ], "backend": "ort", "elements": 1028, "expected_dtype": "float32", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 9.5367431640625e-07, "max_budget_ratio": 0.011430180014133645, "max_rel": 1.1583994209856335e-07, "max_ulp": 1, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } } ], "comparison": { "actual_dtype": "float32", "actual_shape": [ 4, 257 ], "elements": 1028, "expected_dtype": "float32", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 9.5367431640625e-07, "max_budget_ratio": 0.011655980813136606, "max_rel": 1.1800205063963601e-07, "max_ulp": 1, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, "comparison_key": "bbfc094ca80f2ec36ce510f57a20a0482378258f9242f519408dc0d1b01f832e", "contract": "log_probability", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "float32", "shape": [ 4, 257 ] } }, "ir_version": 10, "name": "LogSoftmax/wide/float32/4x257/seed-20260906", "observation_id": "8cbb4a9f6fb31d5ad3b25ce52a8a9afdcf1ee777c7968a5f3205785f6ec98e24", "operation": "LogSoftmax", "operators": [ "LogSoftmax" ], "opset": 19, "optimization": "disabled", "output_hash": "c1f8294a9cf44681e166a42ab9640c9b5a11a2ed141caa08c6c284b101aed60e", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.003125, "p95_ms": 0.0036463, "repeats": 7, "samples_ms": [ 0.003417, 0.003709, 0.0035, 0.003042, 0.003083, 0.002958, 0.003125 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.011917, "p95_ms": 0.0129088, "repeats": 7, "samples_ms": [ 0.013084, 0.0125, 0.012292, 0.011917, 0.011791, 0.011917, 0.011666 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-06, "rtol": 1e-05 }, "variant": "wide", "warnings": [] }, { "artifacts": { "bundle": "observations/f584383ba0d43d72f2c0a879f205a48da349950b0b0ed72aef2f813ae558a8ed/reproducer.zip", "bundle_sha256": "6ff546d6abff6e7593d09acae392753c8ef383a04ce05ace24106305e76f97e2", "case": "artifacts/0b24e238efdde8953e3a09e78ce102a749cd60fabd255ba8f0351f3dcc52ea72/case.json", "inputs": "artifacts/0b24e238efdde8953e3a09e78ce102a749cd60fabd255ba8f0351f3dcc52ea72/inputs.npz", "manifest": "artifacts/0b24e238efdde8953e3a09e78ce102a749cd60fabd255ba8f0351f3dcc52ea72/manifest.json", "model": "artifacts/0b24e238efdde8953e3a09e78ce102a749cd60fabd255ba8f0351f3dcc52ea72/model.onnx", "outputs": "observations/f584383ba0d43d72f2c0a879f205a48da349950b0b0ed72aef2f813ae558a8ed/outputs.npz" }, "candidate_group": "b2bb28dbe77f14f3a6ca54f3ec0f39c3f0f9d5e96b103911cbd6096a5ff223db", "case_id": "0b24e238efdde8953e3a09e78ce102a749cd60fabd255ba8f0351f3dcc52ea72", "category": "softmax", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "backend": "reference", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 257 ], "backend": "reference", "elements": 1028, "expected_dtype": "float64", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 257 ], "backend": "reference", "elements": 1028, "expected_dtype": "float64", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 257 ], "backend": "reference", "elements": 1028, "expected_dtype": "float64", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 8, 257 ], "backend": "reference", "elements": 2056, "expected_dtype": "float64", "expected_shape": [ 8, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 257 ], "backend": "reference", "elements": 1028, "expected_dtype": "float64", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 8.881784197001252e-16, "max_budget_ratio": 0.0002114347007900698, "max_rel": 2.1659073662434864e-16, "max_ulp": 1, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "backend": "ort", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 4.440892098500626e-16, "max_budget_ratio": 0.00040371746350005696, "max_rel": 4.440892098500626e-16, "max_ulp": 2, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 257 ], "backend": "ort", "elements": 1028, "expected_dtype": "float64", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 257 ], "backend": "ort", "elements": 1028, "expected_dtype": "float64", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 257 ], "backend": "ort", "elements": 1028, "expected_dtype": "float64", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 8, 257 ], "backend": "ort", "elements": 2056, "expected_dtype": "float64", "expected_shape": [ 8, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 257 ], "backend": "ort", "elements": 1028, "expected_dtype": "float64", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 8.881784197001252e-16, "max_budget_ratio": 0.00020829721607934797, "max_rel": 2.1329955600582067e-16, "max_ulp": 1, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } } ], "comparison": { "actual_dtype": "float64", "actual_shape": [ 4, 257 ], "elements": 1028, "expected_dtype": "float64", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 1.7763568394002505e-15, "max_budget_ratio": 0.00021920750978967544, "max_rel": 2.2194639168257074e-16, "max_ulp": 1, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, "comparison_key": "a25fc55de2445ccd74aef06c7f1c30d64f374f37d56df3bb9021784f850b02a3", "contract": "log_probability", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "float64", "shape": [ 4, 257 ] } }, "ir_version": 10, "name": "LogSoftmax/wide/float64/4x257/seed-20260906", "observation_id": "f584383ba0d43d72f2c0a879f205a48da349950b0b0ed72aef2f813ae558a8ed", "operation": "LogSoftmax", "operators": [ "LogSoftmax" ], "opset": 19, "optimization": "all", "output_hash": "4a77d38e626a1d4382a8ae4d78f94f427dd16aa947c9fd3c5d633773ec8bae9a", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.004875, "p95_ms": 0.005921299999999999, "repeats": 7, "samples_ms": [ 0.00525, 0.006209, 0.005167, 0.00475, 0.004833, 0.004875, 0.004542 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.0135, "p95_ms": 0.0145957, "repeats": 7, "samples_ms": [ 0.014833, 0.014042, 0.013583, 0.0135, 0.0135, 0.013375, 0.013209 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-13, "rtol": 1e-12 }, "variant": "wide", "warnings": [] }, { "artifacts": { "bundle": "observations/5edab8c2f334084d76b50dc9424a6e5c85b52ea773225d8867263a198a3e612b/reproducer.zip", "bundle_sha256": "30699d0357e86d7b50b090099acfd54b666ce29386f19c5ac55f19ea2cd504cd", "case": "artifacts/0b24e238efdde8953e3a09e78ce102a749cd60fabd255ba8f0351f3dcc52ea72/case.json", "inputs": "artifacts/0b24e238efdde8953e3a09e78ce102a749cd60fabd255ba8f0351f3dcc52ea72/inputs.npz", "manifest": "artifacts/0b24e238efdde8953e3a09e78ce102a749cd60fabd255ba8f0351f3dcc52ea72/manifest.json", "model": "artifacts/0b24e238efdde8953e3a09e78ce102a749cd60fabd255ba8f0351f3dcc52ea72/model.onnx", "outputs": "observations/5edab8c2f334084d76b50dc9424a6e5c85b52ea773225d8867263a198a3e612b/outputs.npz" }, "candidate_group": "b2bb28dbe77f14f3a6ca54f3ec0f39c3f0f9d5e96b103911cbd6096a5ff223db", "case_id": "0b24e238efdde8953e3a09e78ce102a749cd60fabd255ba8f0351f3dcc52ea72", "category": "softmax", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "backend": "reference", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 257 ], "backend": "reference", "elements": 1028, "expected_dtype": "float64", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 257 ], "backend": "reference", "elements": 1028, "expected_dtype": "float64", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 257 ], "backend": "reference", "elements": 1028, "expected_dtype": "float64", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 8, 257 ], "backend": "reference", "elements": 2056, "expected_dtype": "float64", "expected_shape": [ 8, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 257 ], "backend": "reference", "elements": 1028, "expected_dtype": "float64", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 8.881784197001252e-16, "max_budget_ratio": 0.0002114347007900698, "max_rel": 2.1659073662434864e-16, "max_ulp": 1, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "backend": "ort", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 4.440892098500626e-16, "max_budget_ratio": 0.00040371746350005696, "max_rel": 4.440892098500626e-16, "max_ulp": 2, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 257 ], "backend": "ort", "elements": 1028, "expected_dtype": "float64", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 257 ], "backend": "ort", "elements": 1028, "expected_dtype": "float64", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 257 ], "backend": "ort", "elements": 1028, "expected_dtype": "float64", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 8, 257 ], "backend": "ort", "elements": 2056, "expected_dtype": "float64", "expected_shape": [ 8, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 257 ], "backend": "ort", "elements": 1028, "expected_dtype": "float64", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 8.881784197001252e-16, "max_budget_ratio": 0.00020829721607934797, "max_rel": 2.1329955600582067e-16, "max_ulp": 1, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } } ], "comparison": { "actual_dtype": "float64", "actual_shape": [ 4, 257 ], "elements": 1028, "expected_dtype": "float64", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 1.7763568394002505e-15, "max_budget_ratio": 0.00021920750978967544, "max_rel": 2.2194639168257074e-16, "max_ulp": 1, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, "comparison_key": "3526630825e6dbbde9a43ca7bd7e0e5271380b3053c2725fa9f50b36c8a6746b", "contract": "log_probability", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "float64", "shape": [ 4, 257 ] } }, "ir_version": 10, "name": "LogSoftmax/wide/float64/4x257/seed-20260906", "observation_id": "5edab8c2f334084d76b50dc9424a6e5c85b52ea773225d8867263a198a3e612b", "operation": "LogSoftmax", "operators": [ "LogSoftmax" ], "opset": 19, "optimization": "disabled", "output_hash": "4a77d38e626a1d4382a8ae4d78f94f427dd16aa947c9fd3c5d633773ec8bae9a", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.004875, "p95_ms": 0.006583099999999999, "repeats": 7, "samples_ms": [ 0.005041, 0.007208, 0.005125, 0.004875, 0.00475, 0.0045, 0.004583 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.01375, "p95_ms": 0.0147162, "repeats": 7, "samples_ms": [ 0.014916, 0.01425, 0.014125, 0.013542, 0.013458, 0.01375, 0.013292 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-13, "rtol": 1e-12 }, "variant": "wide", "warnings": [] }, { "artifacts": { "bundle": "observations/a2e5e24ea89b1c406ad40612c48cccbe7b743e69c9a83cdec5e1e86e8e86767d/reproducer.zip", "bundle_sha256": "3117f48ae402a6c1b96ce33ee6e460875e3eb52dd3e59cd1b7b35a17e07ea627", "case": "artifacts/98031f8e4f5ae068f58d80186a9f774ab2ba7926309bb33ddc28c9ab17c49091/case.json", "inputs": "artifacts/98031f8e4f5ae068f58d80186a9f774ab2ba7926309bb33ddc28c9ab17c49091/inputs.npz", "manifest": "artifacts/98031f8e4f5ae068f58d80186a9f774ab2ba7926309bb33ddc28c9ab17c49091/manifest.json", "model": "artifacts/98031f8e4f5ae068f58d80186a9f774ab2ba7926309bb33ddc28c9ab17c49091/model.onnx", "outputs": "observations/a2e5e24ea89b1c406ad40612c48cccbe7b743e69c9a83cdec5e1e86e8e86767d/outputs.npz" }, "candidate_group": "4fb6575b97ce4177b49551a6a2a7585bcaed4509063101da536922cba623eb05", "case_id": "98031f8e4f5ae068f58d80186a9f774ab2ba7926309bb33ddc28c9ab17c49091", "category": "normalization", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "scaled_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 7.457008084621464e-05, "max_budget_ratio": 0.01355819651749357, "max_rel": 7.457008084621464e-05, "max_ulp": 335833788312, "mismatched": 0, "name": "l2_unit_or_zero", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 8, 17 ], "backend": "reference", "elements": 136, "expected_dtype": "float16", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "scaled_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 7.457008084621464e-05, "max_budget_ratio": 0.01355819651749357, "max_rel": 7.457008084621464e-05, "max_ulp": 335833788312, "mismatched": 0, "name": "l2_unit_or_zero", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 8, 17 ], "backend": "ort", "elements": 136, "expected_dtype": "float16", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } } ], "comparison": { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, "comparison_key": "04b9f7911a533a3e13bc35667584ba71c5aad64d2749d8802051bbb1d86ca498", "contract": "unit_norm", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "float16", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "LpNormalization/constant/float16/4x17/seed-20260906", "observation_id": "a2e5e24ea89b1c406ad40612c48cccbe7b743e69c9a83cdec5e1e86e8e86767d", "operation": "LpNormalization", "operators": [ "LpNormalization" ], "opset": 19, "optimization": "all", "output_hash": "0fc18eb7a12ab00b49b39c848a32d98ad1492d36e272a75c31b6a113254726c7", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.003375, "p95_ms": 0.0037163, "repeats": 7, "samples_ms": [ 0.003542, 0.003791, 0.003459, 0.003208, 0.003375, 0.003375, 0.003334 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.009208, "p95_ms": 0.0098707, "repeats": 7, "samples_ms": [ 0.009958, 0.009667, 0.009208, 0.009208, 0.008917, 0.00925, 0.008792 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 0.0005, "rtol": 0.005 }, "variant": "constant", "warnings": [] }, { "artifacts": { "bundle": "observations/fb0915b427af7a0bcae746c92a95cee58628558b1fad6cc0b18c63a16097af32/reproducer.zip", "bundle_sha256": "a7e685f41c37b9b2be018e4bf4d05c8408215f7a0f576b5a08dcc701b0a76a3e", "case": "artifacts/98031f8e4f5ae068f58d80186a9f774ab2ba7926309bb33ddc28c9ab17c49091/case.json", "inputs": "artifacts/98031f8e4f5ae068f58d80186a9f774ab2ba7926309bb33ddc28c9ab17c49091/inputs.npz", "manifest": "artifacts/98031f8e4f5ae068f58d80186a9f774ab2ba7926309bb33ddc28c9ab17c49091/manifest.json", "model": "artifacts/98031f8e4f5ae068f58d80186a9f774ab2ba7926309bb33ddc28c9ab17c49091/model.onnx", "outputs": "observations/fb0915b427af7a0bcae746c92a95cee58628558b1fad6cc0b18c63a16097af32/outputs.npz" }, "candidate_group": "4fb6575b97ce4177b49551a6a2a7585bcaed4509063101da536922cba623eb05", "case_id": "98031f8e4f5ae068f58d80186a9f774ab2ba7926309bb33ddc28c9ab17c49091", "category": "normalization", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "scaled_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 7.457008084621464e-05, "max_budget_ratio": 0.01355819651749357, "max_rel": 7.457008084621464e-05, "max_ulp": 335833788312, "mismatched": 0, "name": "l2_unit_or_zero", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 8, 17 ], "backend": "reference", "elements": 136, "expected_dtype": "float16", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "scaled_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 7.457008084621464e-05, "max_budget_ratio": 0.01355819651749357, "max_rel": 7.457008084621464e-05, "max_ulp": 335833788312, "mismatched": 0, "name": "l2_unit_or_zero", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 8, 17 ], "backend": "ort", "elements": 136, "expected_dtype": "float16", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } } ], "comparison": { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, "comparison_key": "6d8f3af32059ca3ef37139da81f484d0a81fd450b6c4011e461d56a9b870156c", "contract": "unit_norm", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "float16", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "LpNormalization/constant/float16/4x17/seed-20260906", "observation_id": "fb0915b427af7a0bcae746c92a95cee58628558b1fad6cc0b18c63a16097af32", "operation": "LpNormalization", "operators": [ "LpNormalization" ], "opset": 19, "optimization": "disabled", "output_hash": "0fc18eb7a12ab00b49b39c848a32d98ad1492d36e272a75c31b6a113254726c7", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.003542, "p95_ms": 0.0038249, "repeats": 7, "samples_ms": [ 0.003875, 0.003667, 0.003708, 0.003417, 0.00325, 0.003542, 0.0035 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.009041, "p95_ms": 0.0099252, "repeats": 7, "samples_ms": [ 0.010125, 0.009459, 0.009291, 0.009041, 0.008958, 0.00875, 0.008583 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 0.0005, "rtol": 0.005 }, "variant": "constant", "warnings": [] }, { "artifacts": { "bundle": "observations/a09a7be67f0c33bc8fceea6d1b37314ffe1367cf513c1a44942fb847d9ed3cd7/reproducer.zip", "bundle_sha256": "b16f0a2f6ce1b1ddfc72a09e3ab8f83d5d665444ad3991395d507bbdb152cec8", "case": "artifacts/81027172143985947574556713fdaa31f0fdb9d365bae06cb2380bd9a230e765/case.json", "inputs": "artifacts/81027172143985947574556713fdaa31f0fdb9d365bae06cb2380bd9a230e765/inputs.npz", "manifest": "artifacts/81027172143985947574556713fdaa31f0fdb9d365bae06cb2380bd9a230e765/manifest.json", "model": "artifacts/81027172143985947574556713fdaa31f0fdb9d365bae06cb2380bd9a230e765/model.onnx", "outputs": "observations/a09a7be67f0c33bc8fceea6d1b37314ffe1367cf513c1a44942fb847d9ed3cd7/outputs.npz" }, "candidate_group": "218772ab5ad3a69b78beb64e1965d145281c64916a4d1c4479b9f40ad7022f68", "case_id": "81027172143985947574556713fdaa31f0fdb9d365bae06cb2380bd9a230e765", "category": "normalization", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "scaled_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 1.69398713945057e-08, "max_budget_ratio": 0.001539988308591427, "max_rel": 1.69398713945057e-08, "max_ulp": 152580797, "mismatched": 0, "name": "l2_unit_or_zero", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 17 ], "backend": "reference", "elements": 136, "expected_dtype": "float32", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "scaled_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 1.69398713945057e-08, "max_budget_ratio": 0.001539988308591427, "max_rel": 1.69398713945057e-08, "max_ulp": 152580797, "mismatched": 0, "name": "l2_unit_or_zero", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 17 ], "backend": "ort", "elements": 136, "expected_dtype": "float32", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } } ], "comparison": { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, "comparison_key": "1014ac62cb6d6b81a7923ce396086e300839e86d545daff71574d796db00f8e0", "contract": "unit_norm", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "float32", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "LpNormalization/constant/float32/4x17/seed-20260906", "observation_id": "a09a7be67f0c33bc8fceea6d1b37314ffe1367cf513c1a44942fb847d9ed3cd7", "operation": "LpNormalization", "operators": [ "LpNormalization" ], "opset": 19, "optimization": "all", "output_hash": "06306ec2fbf88907a1701bc213fffd8d9c100815acd9b1d9009b4ab32b74bfbd", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.002917, "p95_ms": 0.0032212, "repeats": 7, "samples_ms": [ 0.002917, 0.003334, 0.002958, 0.002917, 0.002417, 0.002666, 0.0025 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.008416, "p95_ms": 0.0094413, "repeats": 7, "samples_ms": [ 0.009666, 0.008917, 0.0085, 0.008416, 0.008209, 0.008125, 0.008083 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-06, "rtol": 1e-05 }, "variant": "constant", "warnings": [] }, { "artifacts": { "bundle": "observations/936a59a221774367c9a99f3d927fdd4e6c26af7f53b8466925121ef001aa3e4d/reproducer.zip", "bundle_sha256": "b6f6413362a6e8582419c4509418d125358334370e0153a2153c14d6e0724173", "case": "artifacts/81027172143985947574556713fdaa31f0fdb9d365bae06cb2380bd9a230e765/case.json", "inputs": "artifacts/81027172143985947574556713fdaa31f0fdb9d365bae06cb2380bd9a230e765/inputs.npz", "manifest": "artifacts/81027172143985947574556713fdaa31f0fdb9d365bae06cb2380bd9a230e765/manifest.json", "model": "artifacts/81027172143985947574556713fdaa31f0fdb9d365bae06cb2380bd9a230e765/model.onnx", "outputs": "observations/936a59a221774367c9a99f3d927fdd4e6c26af7f53b8466925121ef001aa3e4d/outputs.npz" }, "candidate_group": "218772ab5ad3a69b78beb64e1965d145281c64916a4d1c4479b9f40ad7022f68", "case_id": "81027172143985947574556713fdaa31f0fdb9d365bae06cb2380bd9a230e765", "category": "normalization", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "scaled_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 1.69398713945057e-08, "max_budget_ratio": 0.001539988308591427, "max_rel": 1.69398713945057e-08, "max_ulp": 152580797, "mismatched": 0, "name": "l2_unit_or_zero", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 17 ], "backend": "reference", "elements": 136, "expected_dtype": "float32", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "scaled_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 1.69398713945057e-08, "max_budget_ratio": 0.001539988308591427, "max_rel": 1.69398713945057e-08, "max_ulp": 152580797, "mismatched": 0, "name": "l2_unit_or_zero", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 17 ], "backend": "ort", "elements": 136, "expected_dtype": "float32", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } } ], "comparison": { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, "comparison_key": "14d6849ecb9e0fa7131019e05b407d6eaa6872b6e0c7cf9375640fff32b272af", "contract": "unit_norm", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "float32", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "LpNormalization/constant/float32/4x17/seed-20260906", "observation_id": "936a59a221774367c9a99f3d927fdd4e6c26af7f53b8466925121ef001aa3e4d", "operation": "LpNormalization", "operators": [ "LpNormalization" ], "opset": 19, "optimization": "disabled", "output_hash": "06306ec2fbf88907a1701bc213fffd8d9c100815acd9b1d9009b4ab32b74bfbd", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.002791, "p95_ms": 0.0033001999999999997, "repeats": 7, "samples_ms": [ 0.00275, 0.0035, 0.002834, 0.002791, 0.002666, 0.002834, 0.002583 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.008667, "p95_ms": 0.0096917, "repeats": 7, "samples_ms": [ 0.009917, 0.009166, 0.008708, 0.008541, 0.008333, 0.008667, 0.008208 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-06, "rtol": 1e-05 }, "variant": "constant", "warnings": [] }, { "artifacts": { "bundle": "observations/6fef0e1f02a319af96d57c4fef9e23c7a0a5baf1c8b259f615730c2a719a7d9e/reproducer.zip", "bundle_sha256": "0920369d173acb4da447c302b9b55d4a6af3d456ee1793cfec7520f98d282aa7", "case": "artifacts/add0fd6706fff31b9ee240fd7fb06bdc9b657fa1f8b7a46aafafaebd4c329fba/case.json", "inputs": "artifacts/add0fd6706fff31b9ee240fd7fb06bdc9b657fa1f8b7a46aafafaebd4c329fba/inputs.npz", "manifest": "artifacts/add0fd6706fff31b9ee240fd7fb06bdc9b657fa1f8b7a46aafafaebd4c329fba/manifest.json", "model": "artifacts/add0fd6706fff31b9ee240fd7fb06bdc9b657fa1f8b7a46aafafaebd4c329fba/model.onnx", "outputs": "observations/6fef0e1f02a319af96d57c4fef9e23c7a0a5baf1c8b259f615730c2a719a7d9e/outputs.npz" }, "candidate_group": "8513450381ef31707b8d3b6a82c9dbe6984efa2e95161ce867ca251f8d441d86", "case_id": "add0fd6706fff31b9ee240fd7fb06bdc9b657fa1f8b7a46aafafaebd4c329fba", "category": "normalization", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 2.7755575615628914e-17, "max_budget_ratio": 8.102974869456237e-05, "max_rel": 1.1443916996305594e-16, "max_ulp": 1, "mismatched": 0, "name": "scaled_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "l2_unit_or_zero", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 8, 17 ], "backend": "reference", "elements": 136, "expected_dtype": "float64", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 2.7755575615628914e-17, "max_budget_ratio": 8.102974869456237e-05, "max_rel": 1.1443916996305594e-16, "max_ulp": 1, "mismatched": 0, "name": "scaled_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "l2_unit_or_zero", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 8, 17 ], "backend": "ort", "elements": 136, "expected_dtype": "float64", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } } ], "comparison": { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, "comparison_key": "04be95cca07365a7f7d72dbd158b849165f6470d0d2592d52e3f4e66e00bfc04", "contract": "unit_norm", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "float64", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "LpNormalization/constant/float64/4x17/seed-20260906", "observation_id": "6fef0e1f02a319af96d57c4fef9e23c7a0a5baf1c8b259f615730c2a719a7d9e", "operation": "LpNormalization", "operators": [ "LpNormalization" ], "opset": 19, "optimization": "all", "output_hash": "24a6e1001d1b146d4753b92f59a3f72786a69a6072386315b0924b151c20b9cc", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.003041, "p95_ms": 0.003333, "repeats": 7, "samples_ms": [ 0.003125, 0.003333, 0.002916, 0.002917, 0.003333, 0.003041, 0.003 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.008583, "p95_ms": 0.0095667, "repeats": 7, "samples_ms": [ 0.009792, 0.009041, 0.008667, 0.008583, 0.008375, 0.0085, 0.00825 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-13, "rtol": 1e-12 }, "variant": "constant", "warnings": [] }, { "artifacts": { "bundle": "observations/56a0c7754a8eae5ca3eeb1f8660ac62df9c4e7303b6eb6f6ee06cda6d7029ee5/reproducer.zip", "bundle_sha256": "08e3c1ea304be56d09c9b4f0794a38bb00df496e847b794bc526489c96c86f1d", "case": "artifacts/add0fd6706fff31b9ee240fd7fb06bdc9b657fa1f8b7a46aafafaebd4c329fba/case.json", "inputs": "artifacts/add0fd6706fff31b9ee240fd7fb06bdc9b657fa1f8b7a46aafafaebd4c329fba/inputs.npz", "manifest": "artifacts/add0fd6706fff31b9ee240fd7fb06bdc9b657fa1f8b7a46aafafaebd4c329fba/manifest.json", "model": "artifacts/add0fd6706fff31b9ee240fd7fb06bdc9b657fa1f8b7a46aafafaebd4c329fba/model.onnx", "outputs": "observations/56a0c7754a8eae5ca3eeb1f8660ac62df9c4e7303b6eb6f6ee06cda6d7029ee5/outputs.npz" }, "candidate_group": "8513450381ef31707b8d3b6a82c9dbe6984efa2e95161ce867ca251f8d441d86", "case_id": "add0fd6706fff31b9ee240fd7fb06bdc9b657fa1f8b7a46aafafaebd4c329fba", "category": "normalization", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 2.7755575615628914e-17, "max_budget_ratio": 8.102974869456237e-05, "max_rel": 1.1443916996305594e-16, "max_ulp": 1, "mismatched": 0, "name": "scaled_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "l2_unit_or_zero", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 8, 17 ], "backend": "reference", "elements": 136, "expected_dtype": "float64", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 2.7755575615628914e-17, "max_budget_ratio": 8.102974869456237e-05, "max_rel": 1.1443916996305594e-16, "max_ulp": 1, "mismatched": 0, "name": "scaled_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "l2_unit_or_zero", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 8, 17 ], "backend": "ort", "elements": 136, "expected_dtype": "float64", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } } ], "comparison": { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, "comparison_key": "8a65f7e7c52d874033246ea582c149c4d3e563375e66451aa2b0bcd111d89332", "contract": "unit_norm", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "float64", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "LpNormalization/constant/float64/4x17/seed-20260906", "observation_id": "56a0c7754a8eae5ca3eeb1f8660ac62df9c4e7303b6eb6f6ee06cda6d7029ee5", "operation": "LpNormalization", "operators": [ "LpNormalization" ], "opset": 19, "optimization": "disabled", "output_hash": "24a6e1001d1b146d4753b92f59a3f72786a69a6072386315b0924b151c20b9cc", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.002792, "p95_ms": 0.0033785, "repeats": 7, "samples_ms": [ 0.002875, 0.003416, 0.003291, 0.002792, 0.002584, 0.002541, 0.002625 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.008292, "p95_ms": 0.009212699999999999, "repeats": 7, "samples_ms": [ 0.009375, 0.008834, 0.008459, 0.008292, 0.008042, 0.00825, 0.007875 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-13, "rtol": 1e-12 }, "variant": "constant", "warnings": [] }, { "artifacts": { "bundle": "observations/cb503617ae51bf832c3bcac5716a533142fb6cf6cee2589479d00e4083eb068c/reproducer.zip", "bundle_sha256": "987151807849b04edd6ed4915e732e0b67141d6b92b68f29bb68213192246af1", "case": "artifacts/49db6cd40097be38311d45dac965498978848ba8462d8017ab1f30ea67f53f0a/case.json", "inputs": "artifacts/49db6cd40097be38311d45dac965498978848ba8462d8017ab1f30ea67f53f0a/inputs.npz", "manifest": "artifacts/49db6cd40097be38311d45dac965498978848ba8462d8017ab1f30ea67f53f0a/manifest.json", "model": "artifacts/49db6cd40097be38311d45dac965498978848ba8462d8017ab1f30ea67f53f0a/model.onnx", "outputs": "observations/cb503617ae51bf832c3bcac5716a533142fb6cf6cee2589479d00e4083eb068c/outputs.npz" }, "candidate_group": "66e66d7a48b64ce79acf6c801f10d270ad9286aa7d0d6d9406d77a33fcad71c1", "case_id": "49db6cd40097be38311d45dac965498978848ba8462d8017ab1f30ea67f53f0a", "category": "normalization", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": [ 0, 0 ], "max_abs": 0.2431640625, "max_budget_ratio": 141.7188389299943, "max_rel": 1.0, "max_ulp": 13256, "mismatched": 68, "name": "scaled_float64_oracle", "nonfinite": 0, "numerical_divergence": true, "passed": false, "reason": "tolerance_exceeded", "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": [ 0 ], "max_abs": 1.0, "max_budget_ratio": 181.81818181818184, "max_rel": 1.0, "max_ulp": 4607182418800017408, "mismatched": 4, "name": "l2_unit_or_zero", "nonfinite": 0, "numerical_divergence": true, "passed": false, "reason": "tolerance_exceeded", "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 8, 17 ], "backend": "reference", "elements": 136, "expected_dtype": "float16", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "scaled_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 0.00010233896113021057, "max_budget_ratio": 0.01860708384185647, "max_rel": 0.00010233896113021057, "max_ulp": 921787414423, "mismatched": 0, "name": "l2_unit_or_zero", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 8, 17 ], "backend": "ort", "elements": 136, "expected_dtype": "float16", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } } ], "comparison": { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": [ 0, 0 ], "max_abs": 0.2431640625, "max_budget_ratio": 486.328125, "max_rel": 1.0928359145037028e+307, "max_ulp": 13256, "mismatched": 68, "nonfinite": 0, "numerical_divergence": true, "passed": false, "reason": "tolerance_exceeded", "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, "comparison_key": "76cc1282f98c38be5bd5826a02991c4377a2ab907e9c6b76111a2462b31df842", "contract": "unit_norm", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "float16", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "LpNormalization/large_offset/float16/4x17/seed-20260906", "observation_id": "cb503617ae51bf832c3bcac5716a533142fb6cf6cee2589479d00e4083eb068c", "operation": "LpNormalization", "operators": [ "LpNormalization" ], "opset": 19, "optimization": "all", "output_hash": "a0be10ecfd4cf9ec4fa71f8ed0862d7eaf61a8192c2d4d94be8dc1abc6d3be9e", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "divergence", "timings": { "ort": { "median_ms": 0.003375, "p95_ms": 0.0039464, "repeats": 7, "samples_ms": [ 0.003917, 0.003959, 0.003375, 0.003042, 0.003042, 0.003583, 0.003166 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.010625, "p95_ms": 0.0118751, "repeats": 7, "samples_ms": [ 0.012125, 0.011083, 0.011292, 0.010625, 0.009834, 0.009833, 0.009917 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 0.0005, "rtol": 0.005 }, "variant": "large_offset", "warnings": [ { "backend": "reference", "message": "overflow encountered in reduce" } ] }, { "artifacts": { "bundle": "observations/0e1fbf54b161392316988e49347125f5f044ba07fb143316ff85f1a9a5e31c29/reproducer.zip", "bundle_sha256": "b228eca8f0b700a3ae54e112d5dae71ccf2dc3fb06a7ef6e25dc80d1575f5df9", "case": "artifacts/49db6cd40097be38311d45dac965498978848ba8462d8017ab1f30ea67f53f0a/case.json", "inputs": "artifacts/49db6cd40097be38311d45dac965498978848ba8462d8017ab1f30ea67f53f0a/inputs.npz", "manifest": "artifacts/49db6cd40097be38311d45dac965498978848ba8462d8017ab1f30ea67f53f0a/manifest.json", "model": "artifacts/49db6cd40097be38311d45dac965498978848ba8462d8017ab1f30ea67f53f0a/model.onnx", "outputs": "observations/0e1fbf54b161392316988e49347125f5f044ba07fb143316ff85f1a9a5e31c29/outputs.npz" }, "candidate_group": "66e66d7a48b64ce79acf6c801f10d270ad9286aa7d0d6d9406d77a33fcad71c1", "case_id": "49db6cd40097be38311d45dac965498978848ba8462d8017ab1f30ea67f53f0a", "category": "normalization", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": [ 0, 0 ], "max_abs": 0.2431640625, "max_budget_ratio": 141.7188389299943, "max_rel": 1.0, "max_ulp": 13256, "mismatched": 68, "name": "scaled_float64_oracle", "nonfinite": 0, "numerical_divergence": true, "passed": false, "reason": "tolerance_exceeded", "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": [ 0 ], "max_abs": 1.0, "max_budget_ratio": 181.81818181818184, "max_rel": 1.0, "max_ulp": 4607182418800017408, "mismatched": 4, "name": "l2_unit_or_zero", "nonfinite": 0, "numerical_divergence": true, "passed": false, "reason": "tolerance_exceeded", "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 8, 17 ], "backend": "reference", "elements": 136, "expected_dtype": "float16", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "scaled_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 0.00010233896113021057, "max_budget_ratio": 0.01860708384185647, "max_rel": 0.00010233896113021057, "max_ulp": 921787414423, "mismatched": 0, "name": "l2_unit_or_zero", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 8, 17 ], "backend": "ort", "elements": 136, "expected_dtype": "float16", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } } ], "comparison": { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": [ 0, 0 ], "max_abs": 0.2431640625, "max_budget_ratio": 486.328125, "max_rel": 1.0928359145037028e+307, "max_ulp": 13256, "mismatched": 68, "nonfinite": 0, "numerical_divergence": true, "passed": false, "reason": "tolerance_exceeded", "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, "comparison_key": "fbaeeb22d92dbb90b4c996961021c4954b8ac81455505eb08fd586ea9e973ed3", "contract": "unit_norm", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "float16", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "LpNormalization/large_offset/float16/4x17/seed-20260906", "observation_id": "0e1fbf54b161392316988e49347125f5f044ba07fb143316ff85f1a9a5e31c29", "operation": "LpNormalization", "operators": [ "LpNormalization" ], "opset": 19, "optimization": "disabled", "output_hash": "a0be10ecfd4cf9ec4fa71f8ed0862d7eaf61a8192c2d4d94be8dc1abc6d3be9e", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "divergence", "timings": { "ort": { "median_ms": 0.003459, "p95_ms": 0.0039454, "repeats": 7, "samples_ms": [ 0.003916, 0.003958, 0.003459, 0.003208, 0.003166, 0.003792, 0.003375 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.010083, "p95_ms": 0.011708799999999998, "repeats": 7, "samples_ms": [ 0.011959, 0.011125, 0.010542, 0.010083, 0.009833, 0.010042, 0.009959 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 0.0005, "rtol": 0.005 }, "variant": "large_offset", "warnings": [ { "backend": "reference", "message": "overflow encountered in reduce" } ] }, { "artifacts": { "bundle": "observations/52b3a7113b021589a728b4f8a05057f0d70c19d1f9d8147002363ab1cebf0949/reproducer.zip", "bundle_sha256": "9381b93625d7624644f8bc82882dd74e0b5d478fb19ada0ef60aaf62a48af965", "case": "artifacts/83f1dbe2198619b8466f759a7baaa652553650845abd2daaa0cf7a54f228c826/case.json", "inputs": "artifacts/83f1dbe2198619b8466f759a7baaa652553650845abd2daaa0cf7a54f228c826/inputs.npz", "manifest": "artifacts/83f1dbe2198619b8466f759a7baaa652553650845abd2daaa0cf7a54f228c826/manifest.json", "model": "artifacts/83f1dbe2198619b8466f759a7baaa652553650845abd2daaa0cf7a54f228c826/model.onnx", "outputs": "observations/52b3a7113b021589a728b4f8a05057f0d70c19d1f9d8147002363ab1cebf0949/outputs.npz" }, "candidate_group": "dc9b91c80dc25f01fadac8367cdedb484603a176b1c651f81d9c142b35efeade", "case_id": "83f1dbe2198619b8466f759a7baaa652553650845abd2daaa0cf7a54f228c826", "category": "normalization", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 1.4901161193847656e-08, "max_budget_ratio": 0.0043503164729074865, "max_rel": 6.14403573567793e-08, "max_ulp": 1, "mismatched": 0, "name": "scaled_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 3.014432325976202e-08, "max_budget_ratio": 0.0027403930236147285, "max_rel": 3.014432325976202e-08, "max_ulp": 216940355, "mismatched": 0, "name": "l2_unit_or_zero", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 17 ], "backend": "reference", "elements": 136, "expected_dtype": "float32", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 2.9802322387695312e-08, "max_budget_ratio": 0.008700579576927705, "max_rel": 1.228796501960422e-07, "max_ulp": 2, "mismatched": 0, "name": "scaled_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 7.110217636885352e-08, "max_budget_ratio": 0.006463834215350319, "max_rel": 7.110217636885352e-08, "max_ulp": 640431470, "mismatched": 0, "name": "l2_unit_or_zero", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 17 ], "backend": "ort", "elements": 136, "expected_dtype": "float32", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 2.9802322387695312e-08, "max_budget_ratio": 0.00870058033392862, "max_rel": 1.2287966529545248e-07, "max_ulp": 2, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } } ], "comparison": { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 2.9802322387695312e-08, "max_budget_ratio": 0.008700579576927705, "max_rel": 1.228796501960422e-07, "max_ulp": 2, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, "comparison_key": "5e3a6f1fa826b12bbc80744fc7f81eae6ec5894d28d165f7ae963905341183b5", "contract": "unit_norm", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "float32", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "LpNormalization/large_offset/float32/4x17/seed-20260906", "observation_id": "52b3a7113b021589a728b4f8a05057f0d70c19d1f9d8147002363ab1cebf0949", "operation": "LpNormalization", "operators": [ "LpNormalization" ], "opset": 19, "optimization": "all", "output_hash": "3e1aef1e1bec4b130dd675e2600ce5b95a71e4e1f963445b4ca61464d198d366", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.002834, "p95_ms": 0.0031169, "repeats": 7, "samples_ms": [ 0.002834, 0.003167, 0.002958, 0.003, 0.00275, 0.002625, 0.002792 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.008583, "p95_ms": 0.0095, "repeats": 7, "samples_ms": [ 0.0095, 0.008917, 0.008583, 0.0085, 0.008583, 0.0095, 0.008542 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-06, "rtol": 1e-05 }, "variant": "large_offset", "warnings": [] }, { "artifacts": { "bundle": "observations/d33f8e8477d302564ce38fe58f2d77c5c604885a4c120ba7523ba774b2f7bf4e/reproducer.zip", "bundle_sha256": "3c88f182ea06a099ea502b9f1b7f5bea73c6363900674f1516dbb0eadecb5a66", "case": "artifacts/83f1dbe2198619b8466f759a7baaa652553650845abd2daaa0cf7a54f228c826/case.json", "inputs": "artifacts/83f1dbe2198619b8466f759a7baaa652553650845abd2daaa0cf7a54f228c826/inputs.npz", "manifest": "artifacts/83f1dbe2198619b8466f759a7baaa652553650845abd2daaa0cf7a54f228c826/manifest.json", "model": "artifacts/83f1dbe2198619b8466f759a7baaa652553650845abd2daaa0cf7a54f228c826/model.onnx", "outputs": "observations/d33f8e8477d302564ce38fe58f2d77c5c604885a4c120ba7523ba774b2f7bf4e/outputs.npz" }, "candidate_group": "dc9b91c80dc25f01fadac8367cdedb484603a176b1c651f81d9c142b35efeade", "case_id": "83f1dbe2198619b8466f759a7baaa652553650845abd2daaa0cf7a54f228c826", "category": "normalization", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 1.4901161193847656e-08, "max_budget_ratio": 0.0043503164729074865, "max_rel": 6.14403573567793e-08, "max_ulp": 1, "mismatched": 0, "name": "scaled_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 3.014432325976202e-08, "max_budget_ratio": 0.0027403930236147285, "max_rel": 3.014432325976202e-08, "max_ulp": 216940355, "mismatched": 0, "name": "l2_unit_or_zero", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 17 ], "backend": "reference", "elements": 136, "expected_dtype": "float32", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 2.9802322387695312e-08, "max_budget_ratio": 0.008700579576927705, "max_rel": 1.228796501960422e-07, "max_ulp": 2, "mismatched": 0, "name": "scaled_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 7.110217636885352e-08, "max_budget_ratio": 0.006463834215350319, "max_rel": 7.110217636885352e-08, "max_ulp": 640431470, "mismatched": 0, "name": "l2_unit_or_zero", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 17 ], "backend": "ort", "elements": 136, "expected_dtype": "float32", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 2.9802322387695312e-08, "max_budget_ratio": 0.00870058033392862, "max_rel": 1.2287966529545248e-07, "max_ulp": 2, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } } ], "comparison": { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 2.9802322387695312e-08, "max_budget_ratio": 0.008700579576927705, "max_rel": 1.228796501960422e-07, "max_ulp": 2, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, "comparison_key": "8add6248289afd5f62758494e6d18345e414d876f21989120cee4a732fc3f8bf", "contract": "unit_norm", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "float32", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "LpNormalization/large_offset/float32/4x17/seed-20260906", "observation_id": "d33f8e8477d302564ce38fe58f2d77c5c604885a4c120ba7523ba774b2f7bf4e", "operation": "LpNormalization", "operators": [ "LpNormalization" ], "opset": 19, "optimization": "disabled", "output_hash": "3e1aef1e1bec4b130dd675e2600ce5b95a71e4e1f963445b4ca61464d198d366", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.002959, "p95_ms": 0.003675, "repeats": 7, "samples_ms": [ 0.002959, 0.00375, 0.00325, 0.0035, 0.002917, 0.002875, 0.002792 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.008458, "p95_ms": 0.0093296, "repeats": 7, "samples_ms": [ 0.009542, 0.008834, 0.0085, 0.008333, 0.008084, 0.008458, 0.008 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-06, "rtol": 1e-05 }, "variant": "large_offset", "warnings": [] }, { "artifacts": { "bundle": "observations/f350608f2ca1f90649f99cb039806261a5393053f14836c3fafc605817f96650/reproducer.zip", "bundle_sha256": "88397bfb0e2a8da8fdd03ed6abccb77680bbf4b19abdbf99a4d76cd48b8836ea", "case": "artifacts/9ae452469a4524c9fc0bb91882c1fb1589e0ea42ee253ee8b43b175b92f3318f/case.json", "inputs": "artifacts/9ae452469a4524c9fc0bb91882c1fb1589e0ea42ee253ee8b43b175b92f3318f/inputs.npz", "manifest": "artifacts/9ae452469a4524c9fc0bb91882c1fb1589e0ea42ee253ee8b43b175b92f3318f/manifest.json", "model": "artifacts/9ae452469a4524c9fc0bb91882c1fb1589e0ea42ee253ee8b43b175b92f3318f/model.onnx", "outputs": "observations/f350608f2ca1f90649f99cb039806261a5393053f14836c3fafc605817f96650/outputs.npz" }, "candidate_group": "6f0994c8c428046bdc40a4787d6783f1f990220d4e8976a61690f913b7925972", "case_id": "9ae452469a4524c9fc0bb91882c1fb1589e0ea42ee253ee8b43b175b92f3318f", "category": "normalization", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 5.551115123125783e-17, "max_budget_ratio": 0.0001620589274717845, "max_rel": 2.288772031585937e-16, "max_ulp": 2, "mismatched": 0, "name": "scaled_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 1.1102230246251565e-16, "max_budget_ratio": 0.00010092936587501424, "max_rel": 1.1102230246251565e-16, "max_ulp": 1, "mismatched": 0, "name": "l2_unit_or_zero", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 8, 17 ], "backend": "reference", "elements": 136, "expected_dtype": "float64", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 5.551115123125783e-17, "max_budget_ratio": 0.00016206206533543878, "max_rel": 2.288834620395425e-16, "max_ulp": 2, "mismatched": 0, "name": "scaled_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 1.1102230246251565e-16, "max_budget_ratio": 0.00010092936587501424, "max_rel": 1.1102230246251565e-16, "max_ulp": 1, "mismatched": 0, "name": "l2_unit_or_zero", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 8, 17 ], "backend": "ort", "elements": 136, "expected_dtype": "float64", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 5.551115123125783e-17, "max_budget_ratio": 0.00016206196339657423, "max_rel": 2.2888325870752575e-16, "max_ulp": 2, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } } ], "comparison": { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 5.551115123125783e-17, "max_budget_ratio": 0.00016206088816429274, "max_rel": 2.28881114005458e-16, "max_ulp": 2, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, "comparison_key": "a42ab316503768028e408a034afc3bb12bf3354afe8b7065135b91dd98aea97c", "contract": "unit_norm", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "float64", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "LpNormalization/large_offset/float64/4x17/seed-20260906", "observation_id": "f350608f2ca1f90649f99cb039806261a5393053f14836c3fafc605817f96650", "operation": "LpNormalization", "operators": [ "LpNormalization" ], "opset": 19, "optimization": "all", "output_hash": "a63828f1e67c7cfbdd84b430ba0895c3e19dac033070351c5d97cff249bad753", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.003083, "p95_ms": 0.0034876, "repeats": 7, "samples_ms": [ 0.003166, 0.003625, 0.003167, 0.002958, 0.003083, 0.002667, 0.002708 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.008333, "p95_ms": 0.0098296, "repeats": 7, "samples_ms": [ 0.010042, 0.009334, 0.008542, 0.008292, 0.008209, 0.008333, 0.008084 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-13, "rtol": 1e-12 }, "variant": "large_offset", "warnings": [] }, { "artifacts": { "bundle": "observations/aa8b9862951f56c6a8c748977cf8c15dfd2d1c7f51900c78dc0d9a65fb88db01/reproducer.zip", "bundle_sha256": "477c711cf82a3ea6698bd9f3794f29574510534a1c530e712f981715050c9c3b", "case": "artifacts/9ae452469a4524c9fc0bb91882c1fb1589e0ea42ee253ee8b43b175b92f3318f/case.json", "inputs": "artifacts/9ae452469a4524c9fc0bb91882c1fb1589e0ea42ee253ee8b43b175b92f3318f/inputs.npz", "manifest": "artifacts/9ae452469a4524c9fc0bb91882c1fb1589e0ea42ee253ee8b43b175b92f3318f/manifest.json", "model": "artifacts/9ae452469a4524c9fc0bb91882c1fb1589e0ea42ee253ee8b43b175b92f3318f/model.onnx", "outputs": "observations/aa8b9862951f56c6a8c748977cf8c15dfd2d1c7f51900c78dc0d9a65fb88db01/outputs.npz" }, "candidate_group": "6f0994c8c428046bdc40a4787d6783f1f990220d4e8976a61690f913b7925972", "case_id": "9ae452469a4524c9fc0bb91882c1fb1589e0ea42ee253ee8b43b175b92f3318f", "category": "normalization", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 5.551115123125783e-17, "max_budget_ratio": 0.0001620589274717845, "max_rel": 2.288772031585937e-16, "max_ulp": 2, "mismatched": 0, "name": "scaled_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 1.1102230246251565e-16, "max_budget_ratio": 0.00010092936587501424, "max_rel": 1.1102230246251565e-16, "max_ulp": 1, "mismatched": 0, "name": "l2_unit_or_zero", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 8, 17 ], "backend": "reference", "elements": 136, "expected_dtype": "float64", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 5.551115123125783e-17, "max_budget_ratio": 0.00016206206533543878, "max_rel": 2.288834620395425e-16, "max_ulp": 2, "mismatched": 0, "name": "scaled_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 1.1102230246251565e-16, "max_budget_ratio": 0.00010092936587501424, "max_rel": 1.1102230246251565e-16, "max_ulp": 1, "mismatched": 0, "name": "l2_unit_or_zero", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 8, 17 ], "backend": "ort", "elements": 136, "expected_dtype": "float64", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 5.551115123125783e-17, "max_budget_ratio": 0.00016206196339657423, "max_rel": 2.2888325870752575e-16, "max_ulp": 2, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } } ], "comparison": { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 5.551115123125783e-17, "max_budget_ratio": 0.00016206088816429274, "max_rel": 2.28881114005458e-16, "max_ulp": 2, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, "comparison_key": "6ac1370a3f0225ad3344ef6093301454a136b2b4de687ca0343da2c3cd3ce75f", "contract": "unit_norm", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "float64", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "LpNormalization/large_offset/float64/4x17/seed-20260906", "observation_id": "aa8b9862951f56c6a8c748977cf8c15dfd2d1c7f51900c78dc0d9a65fb88db01", "operation": "LpNormalization", "operators": [ "LpNormalization" ], "opset": 19, "optimization": "disabled", "output_hash": "a63828f1e67c7cfbdd84b430ba0895c3e19dac033070351c5d97cff249bad753", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.002791, "p95_ms": 0.0031875999999999996, "repeats": 7, "samples_ms": [ 0.002791, 0.00325, 0.003042, 0.002875, 0.002666, 0.002667, 0.0025 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.008208, "p95_ms": 0.0093829, "repeats": 7, "samples_ms": [ 0.009583, 0.008916, 0.008417, 0.008167, 0.008042, 0.008208, 0.007916 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-13, "rtol": 1e-12 }, "variant": "large_offset", "warnings": [] }, { "artifacts": { "bundle": "observations/77bc2f72ee2f07c541711d862525d18d3d3684ab634520cc99221afa37d7da97/reproducer.zip", "bundle_sha256": "085c7e5fe364e764fdc78407deadd77b1c493b8c1964941ce4a271f20bd5a63f", "case": "artifacts/9e83a423f7eeeaad462bbda2eb943a37d65dac00c1705d4100c7791ddaef6650/case.json", "inputs": "artifacts/9e83a423f7eeeaad462bbda2eb943a37d65dac00c1705d4100c7791ddaef6650/inputs.npz", "manifest": "artifacts/9e83a423f7eeeaad462bbda2eb943a37d65dac00c1705d4100c7791ddaef6650/manifest.json", "model": "artifacts/9e83a423f7eeeaad462bbda2eb943a37d65dac00c1705d4100c7791ddaef6650/model.onnx", "outputs": "observations/77bc2f72ee2f07c541711d862525d18d3d3684ab634520cc99221afa37d7da97/outputs.npz" }, "candidate_group": "bffa4b928fb8f43cb143b588fd90515c429953677b239e00144b138a92869bb7", "case_id": "9e83a423f7eeeaad462bbda2eb943a37d65dac00c1705d4100c7791ddaef6650", "category": "normalization", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.00048828125, "max_budget_ratio": 0.15328019619865113, "max_rel": 0.0009157509157509158, "max_ulp": 1, "mismatched": 0, "name": "scaled_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 0.0006639119084592693, "max_budget_ratio": 0.12071125608350351, "max_rel": 0.0006639119084592693, "max_ulp": 5330838149061, "mismatched": 0, "name": "l2_unit_or_zero", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 8, 17 ], "backend": "reference", "elements": 136, "expected_dtype": "float16", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "scaled_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 0.0001561326836286847, "max_budget_ratio": 0.02838776065976086, "max_rel": 0.0001561326836286847, "max_ulp": 1406318191621, "mismatched": 0, "name": "l2_unit_or_zero", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 8, 17 ], "backend": "ort", "elements": 136, "expected_dtype": "float16", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } } ], "comparison": { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.00048828125, "max_budget_ratio": 0.1531628120692296, "max_rel": 0.0009149130832570906, "max_ulp": 1, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, "comparison_key": "2b6051dc99f8757004f44443ec2c27465039adeddb52f9cb8612b6ed867975b4", "contract": "unit_norm", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "float16", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "LpNormalization/ordinary/float16/4x17/seed-20260906", "observation_id": "77bc2f72ee2f07c541711d862525d18d3d3684ab634520cc99221afa37d7da97", "operation": "LpNormalization", "operators": [ "LpNormalization" ], "opset": 19, "optimization": "all", "output_hash": "bef7ad76ddb928335fd44c24df4550c3a1ffe167ef6b8ce6d4f3de070aa03c69", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.003375, "p95_ms": 0.0036544, "repeats": 7, "samples_ms": [ 0.003625, 0.003667, 0.003458, 0.003083, 0.003125, 0.003375, 0.003292 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.010583, "p95_ms": 0.013583799999999997, "repeats": 7, "samples_ms": [ 0.014584, 0.01125, 0.010583, 0.010125, 0.010583, 0.010459, 0.010125 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 0.0005, "rtol": 0.005 }, "variant": "ordinary", "warnings": [ { "backend": "reference", "message": "underflow encountered in multiply" }, { "backend": "reference", "message": "underflow encountered in power" } ] }, { "artifacts": { "bundle": "observations/4eb05314309810552b708dcccbdd3c4f9c968362db86946ef80a39f42c40e515/reproducer.zip", "bundle_sha256": "48b2c588531297ede3d8515b875c75e390efe9dedc2677fb94b638269202b41c", "case": "artifacts/9e83a423f7eeeaad462bbda2eb943a37d65dac00c1705d4100c7791ddaef6650/case.json", "inputs": "artifacts/9e83a423f7eeeaad462bbda2eb943a37d65dac00c1705d4100c7791ddaef6650/inputs.npz", "manifest": "artifacts/9e83a423f7eeeaad462bbda2eb943a37d65dac00c1705d4100c7791ddaef6650/manifest.json", "model": "artifacts/9e83a423f7eeeaad462bbda2eb943a37d65dac00c1705d4100c7791ddaef6650/model.onnx", "outputs": "observations/4eb05314309810552b708dcccbdd3c4f9c968362db86946ef80a39f42c40e515/outputs.npz" }, "candidate_group": "bffa4b928fb8f43cb143b588fd90515c429953677b239e00144b138a92869bb7", "case_id": "9e83a423f7eeeaad462bbda2eb943a37d65dac00c1705d4100c7791ddaef6650", "category": "normalization", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.00048828125, "max_budget_ratio": 0.15328019619865113, "max_rel": 0.0009157509157509158, "max_ulp": 1, "mismatched": 0, "name": "scaled_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 0.0006639119084592693, "max_budget_ratio": 0.12071125608350351, "max_rel": 0.0006639119084592693, "max_ulp": 5330838149061, "mismatched": 0, "name": "l2_unit_or_zero", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 8, 17 ], "backend": "reference", "elements": 136, "expected_dtype": "float16", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "scaled_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 0.0001561326836286847, "max_budget_ratio": 0.02838776065976086, "max_rel": 0.0001561326836286847, "max_ulp": 1406318191621, "mismatched": 0, "name": "l2_unit_or_zero", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 8, 17 ], "backend": "ort", "elements": 136, "expected_dtype": "float16", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } } ], "comparison": { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.00048828125, "max_budget_ratio": 0.1531628120692296, "max_rel": 0.0009149130832570906, "max_ulp": 1, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, "comparison_key": "4f9e76601e7f09dd7174a6239f856268d614b24c1c568d762ecf2bdeeca7353d", "contract": "unit_norm", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "float16", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "LpNormalization/ordinary/float16/4x17/seed-20260906", "observation_id": "4eb05314309810552b708dcccbdd3c4f9c968362db86946ef80a39f42c40e515", "operation": "LpNormalization", "operators": [ "LpNormalization" ], "opset": 19, "optimization": "disabled", "output_hash": "bef7ad76ddb928335fd44c24df4550c3a1ffe167ef6b8ce6d4f3de070aa03c69", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.00375, "p95_ms": 0.0041705, "repeats": 7, "samples_ms": [ 0.004083, 0.004, 0.00375, 0.004208, 0.00325, 0.003666, 0.003458 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.010625, "p95_ms": 0.0125787, "repeats": 7, "samples_ms": [ 0.013041, 0.0115, 0.011125, 0.010459, 0.010208, 0.010625, 0.010375 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 0.0005, "rtol": 0.005 }, "variant": "ordinary", "warnings": [ { "backend": "reference", "message": "underflow encountered in multiply" }, { "backend": "reference", "message": "underflow encountered in power" } ] }, { "artifacts": { "bundle": "observations/3c0ec02a145a866b236714964deac751f1fc4225ed9aa299803f2df43aff3b1b/reproducer.zip", "bundle_sha256": "08d736b982c584857da679f15432c7418bfacd394a23eef426870b5f31d3a7fa", "case": "artifacts/22b52643cf763a3cf081648fae6cc6ca0bc290f0da96b5a532e67f2aea3b4739/case.json", "inputs": "artifacts/22b52643cf763a3cf081648fae6cc6ca0bc290f0da96b5a532e67f2aea3b4739/inputs.npz", "manifest": "artifacts/22b52643cf763a3cf081648fae6cc6ca0bc290f0da96b5a532e67f2aea3b4739/manifest.json", "model": "artifacts/22b52643cf763a3cf081648fae6cc6ca0bc290f0da96b5a532e67f2aea3b4739/model.onnx", "outputs": "observations/3c0ec02a145a866b236714964deac751f1fc4225ed9aa299803f2df43aff3b1b/outputs.npz" }, "candidate_group": "e31c06ead18335d715033cf3cbf34bff28bc4ce838ce5c4e5cdf0b29e9472869", "case_id": "22b52643cf763a3cf081648fae6cc6ca0bc290f0da96b5a532e67f2aea3b4739", "category": "normalization", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 2.9802322387695312e-08, "max_budget_ratio": 0.007254135477450729, "max_rel": 1.1817675484504014e-07, "max_ulp": 1, "mismatched": 0, "name": "scaled_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 4.7714167283885445e-08, "max_budget_ratio": 0.004337651571262312, "max_rel": 4.7714167283885445e-08, "max_ulp": 422361030, "mismatched": 0, "name": "l2_unit_or_zero", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 17 ], "backend": "reference", "elements": 136, "expected_dtype": "float32", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 5.960464477539063e-08, "max_budget_ratio": 0.010752498326857497, "max_rel": 1.682164730691314e-07, "max_ulp": 2, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 5.960464477539063e-08, "max_budget_ratio": 0.008161721841239394, "max_rel": 1.18594461140568e-07, "max_ulp": 1, "mismatched": 0, "name": "scaled_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 5.8596788754528006e-08, "max_budget_ratio": 0.005326980795866181, "max_rel": 5.8596788754528006e-08, "max_ulp": 422361030, "mismatched": 0, "name": "l2_unit_or_zero", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 17 ], "backend": "ort", "elements": 136, "expected_dtype": "float32", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 5.960464477539063e-08, "max_budget_ratio": 0.01075249717069542, "max_rel": 1.6821644477235437e-07, "max_ulp": 2, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } } ], "comparison": { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 5.960464477539063e-08, "max_budget_ratio": 0.010752498326857497, "max_rel": 1.682164730691314e-07, "max_ulp": 2, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, "comparison_key": "59eaf1743c732495fa4b0c4e9fd37982ac629c186c2266f1b8d1d4adbf8a5f3c", "contract": "unit_norm", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "float32", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "LpNormalization/ordinary/float32/4x17/seed-20260906", "observation_id": "3c0ec02a145a866b236714964deac751f1fc4225ed9aa299803f2df43aff3b1b", "operation": "LpNormalization", "operators": [ "LpNormalization" ], "opset": 19, "optimization": "all", "output_hash": "386564ab90aff5af051a9c3a4de21ade69241bf636e17ed723d7173fbe8762b9", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.002958, "p95_ms": 0.0033748999999999997, "repeats": 7, "samples_ms": [ 0.003083, 0.0035, 0.002958, 0.003042, 0.002584, 0.002791, 0.002666 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.009917, "p95_ms": 0.0109499, "repeats": 7, "samples_ms": [ 0.010542, 0.009917, 0.011, 0.010833, 0.008459, 0.008416, 0.008208 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-06, "rtol": 1e-05 }, "variant": "ordinary", "warnings": [] }, { "artifacts": { "bundle": "observations/14293e73647a46c857ca30b3ec17e357ebb7bea730cec7188285712a7d516880/reproducer.zip", "bundle_sha256": "4f4a0dd17217fc7d12de8c06f1c5cf5ee56db188c237922d6e08bf8fb4ed32e6", "case": "artifacts/22b52643cf763a3cf081648fae6cc6ca0bc290f0da96b5a532e67f2aea3b4739/case.json", "inputs": "artifacts/22b52643cf763a3cf081648fae6cc6ca0bc290f0da96b5a532e67f2aea3b4739/inputs.npz", "manifest": "artifacts/22b52643cf763a3cf081648fae6cc6ca0bc290f0da96b5a532e67f2aea3b4739/manifest.json", "model": "artifacts/22b52643cf763a3cf081648fae6cc6ca0bc290f0da96b5a532e67f2aea3b4739/model.onnx", "outputs": "observations/14293e73647a46c857ca30b3ec17e357ebb7bea730cec7188285712a7d516880/outputs.npz" }, "candidate_group": "e31c06ead18335d715033cf3cbf34bff28bc4ce838ce5c4e5cdf0b29e9472869", "case_id": "22b52643cf763a3cf081648fae6cc6ca0bc290f0da96b5a532e67f2aea3b4739", "category": "normalization", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 2.9802322387695312e-08, "max_budget_ratio": 0.007254135477450729, "max_rel": 1.1817675484504014e-07, "max_ulp": 1, "mismatched": 0, "name": "scaled_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 4.7714167283885445e-08, "max_budget_ratio": 0.004337651571262312, "max_rel": 4.7714167283885445e-08, "max_ulp": 422361030, "mismatched": 0, "name": "l2_unit_or_zero", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 17 ], "backend": "reference", "elements": 136, "expected_dtype": "float32", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 5.960464477539063e-08, "max_budget_ratio": 0.010752498326857497, "max_rel": 1.682164730691314e-07, "max_ulp": 2, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 5.960464477539063e-08, "max_budget_ratio": 0.008161721841239394, "max_rel": 1.18594461140568e-07, "max_ulp": 1, "mismatched": 0, "name": "scaled_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 5.8596788754528006e-08, "max_budget_ratio": 0.005326980795866181, "max_rel": 5.8596788754528006e-08, "max_ulp": 422361030, "mismatched": 0, "name": "l2_unit_or_zero", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 17 ], "backend": "ort", "elements": 136, "expected_dtype": "float32", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 5.960464477539063e-08, "max_budget_ratio": 0.01075249717069542, "max_rel": 1.6821644477235437e-07, "max_ulp": 2, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } } ], "comparison": { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 5.960464477539063e-08, "max_budget_ratio": 0.010752498326857497, "max_rel": 1.682164730691314e-07, "max_ulp": 2, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, "comparison_key": "1957fede51e8c977beedc1626177b9b9162985048bab718a315baf021b1c4977", "contract": "unit_norm", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "float32", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "LpNormalization/ordinary/float32/4x17/seed-20260906", "observation_id": "14293e73647a46c857ca30b3ec17e357ebb7bea730cec7188285712a7d516880", "operation": "LpNormalization", "operators": [ "LpNormalization" ], "opset": 19, "optimization": "disabled", "output_hash": "386564ab90aff5af051a9c3a4de21ade69241bf636e17ed723d7173fbe8762b9", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.002958, "p95_ms": 0.0036542999999999996, "repeats": 7, "samples_ms": [ 0.002917, 0.003792, 0.003333, 0.003, 0.002958, 0.002875, 0.002875 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.008791, "p95_ms": 0.0097876, "repeats": 7, "samples_ms": [ 0.01, 0.009292, 0.008791, 0.008583, 0.008416, 0.008834, 0.008292 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-06, "rtol": 1e-05 }, "variant": "ordinary", "warnings": [] }, { "artifacts": { "bundle": "observations/9ad4a1f81c051105014d03fa38a42240a55c7bd97dea1bece5674e71fd23597d/reproducer.zip", "bundle_sha256": "a1e4e6840f1291d65af1c657d974c841f1d4148e47f1cd8986027df96c8c7a9a", "case": "artifacts/c863043334309cf1b052fb8f934ea1dfe50777cccb1490b3c1c5f51c4d921601/case.json", "inputs": "artifacts/c863043334309cf1b052fb8f934ea1dfe50777cccb1490b3c1c5f51c4d921601/inputs.npz", "manifest": "artifacts/c863043334309cf1b052fb8f934ea1dfe50777cccb1490b3c1c5f51c4d921601/manifest.json", "model": "artifacts/c863043334309cf1b052fb8f934ea1dfe50777cccb1490b3c1c5f51c4d921601/model.onnx", "outputs": "observations/9ad4a1f81c051105014d03fa38a42240a55c7bd97dea1bece5674e71fd23597d/outputs.npz" }, "candidate_group": "f1d8d2dac47e5994ecc5e6b4886275f21ff972293e743688a33b6f79ed44606b", "case_id": "c863043334309cf1b052fb8f934ea1dfe50777cccb1490b3c1c5f51c4d921601", "category": "normalization", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 1.1102230246251565e-16, "max_budget_ratio": 0.00018471238595049125, "max_rel": 2.518028940968256e-16, "max_ulp": 2, "mismatched": 0, "name": "scaled_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 1.1102230246251565e-16, "max_budget_ratio": 0.00010092936587501424, "max_rel": 1.1102230246251565e-16, "max_ulp": 1, "mismatched": 0, "name": "l2_unit_or_zero", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 8, 17 ], "backend": "reference", "elements": 136, "expected_dtype": "float64", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 2.220446049250313e-16, "max_budget_ratio": 0.0003694247719009825, "max_rel": 4.431541578157274e-16, "max_ulp": 3, "mismatched": 0, "name": "scaled_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 3.3306690738754696e-16, "max_budget_ratio": 0.0003027880976250427, "max_rel": 3.3306690738754696e-16, "max_ulp": 3, "mismatched": 0, "name": "l2_unit_or_zero", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 8, 17 ], "backend": "ort", "elements": 136, "expected_dtype": "float64", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 1.1102230246251565e-16, "max_budget_ratio": 0.00024645125763842266, "max_rel": 3.167687011031997e-16, "max_ulp": 2, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } } ], "comparison": { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 1.1102230246251565e-16, "max_budget_ratio": 0.00018471238595049128, "max_rel": 3.1327077364602973e-16, "max_ulp": 2, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, "comparison_key": "0a85ce74447e806738d61f43b811cd7178d39ba2907a7c8cf27f6d15d026c6f9", "contract": "unit_norm", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "float64", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "LpNormalization/ordinary/float64/4x17/seed-20260906", "observation_id": "9ad4a1f81c051105014d03fa38a42240a55c7bd97dea1bece5674e71fd23597d", "operation": "LpNormalization", "operators": [ "LpNormalization" ], "opset": 19, "optimization": "all", "output_hash": "01a431ce7361dad4d4563ba12950bf58a8e80057b017e394a1ba62faf636ba50", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.002958, "p95_ms": 0.0031704999999999997, "repeats": 7, "samples_ms": [ 0.003083, 0.003208, 0.002959, 0.002958, 0.00275, 0.002542, 0.002584 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.0085, "p95_ms": 0.0096206, "repeats": 7, "samples_ms": [ 0.009833, 0.009125, 0.008625, 0.0085, 0.00825, 0.008375, 0.008083 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-13, "rtol": 1e-12 }, "variant": "ordinary", "warnings": [] }, { "artifacts": { "bundle": "observations/2634c2fa94d4ce712e43e2737cca99a665ff443367b7730d462d65b374ddf1d2/reproducer.zip", "bundle_sha256": "72183c7185ebac49eea410055ab5211ce3598ecccfe3380ef79e912e679127ee", "case": "artifacts/c863043334309cf1b052fb8f934ea1dfe50777cccb1490b3c1c5f51c4d921601/case.json", "inputs": "artifacts/c863043334309cf1b052fb8f934ea1dfe50777cccb1490b3c1c5f51c4d921601/inputs.npz", "manifest": "artifacts/c863043334309cf1b052fb8f934ea1dfe50777cccb1490b3c1c5f51c4d921601/manifest.json", "model": "artifacts/c863043334309cf1b052fb8f934ea1dfe50777cccb1490b3c1c5f51c4d921601/model.onnx", "outputs": "observations/2634c2fa94d4ce712e43e2737cca99a665ff443367b7730d462d65b374ddf1d2/outputs.npz" }, "candidate_group": "f1d8d2dac47e5994ecc5e6b4886275f21ff972293e743688a33b6f79ed44606b", "case_id": "c863043334309cf1b052fb8f934ea1dfe50777cccb1490b3c1c5f51c4d921601", "category": "normalization", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 1.1102230246251565e-16, "max_budget_ratio": 0.00018471238595049125, "max_rel": 2.518028940968256e-16, "max_ulp": 2, "mismatched": 0, "name": "scaled_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 1.1102230246251565e-16, "max_budget_ratio": 0.00010092936587501424, "max_rel": 1.1102230246251565e-16, "max_ulp": 1, "mismatched": 0, "name": "l2_unit_or_zero", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 8, 17 ], "backend": "reference", "elements": 136, "expected_dtype": "float64", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 2.220446049250313e-16, "max_budget_ratio": 0.0003694247719009825, "max_rel": 4.431541578157274e-16, "max_ulp": 3, "mismatched": 0, "name": "scaled_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 3.3306690738754696e-16, "max_budget_ratio": 0.0003027880976250427, "max_rel": 3.3306690738754696e-16, "max_ulp": 3, "mismatched": 0, "name": "l2_unit_or_zero", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 8, 17 ], "backend": "ort", "elements": 136, "expected_dtype": "float64", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 1.1102230246251565e-16, "max_budget_ratio": 0.00024645125763842266, "max_rel": 3.167687011031997e-16, "max_ulp": 2, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } } ], "comparison": { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 1.1102230246251565e-16, "max_budget_ratio": 0.00018471238595049128, "max_rel": 3.1327077364602973e-16, "max_ulp": 2, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, "comparison_key": "a51de523b2367339f26ece42102171f52e58d9a5d49168f4d146425246c65124", "contract": "unit_norm", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "float64", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "LpNormalization/ordinary/float64/4x17/seed-20260906", "observation_id": "2634c2fa94d4ce712e43e2737cca99a665ff443367b7730d462d65b374ddf1d2", "operation": "LpNormalization", "operators": [ "LpNormalization" ], "opset": 19, "optimization": "disabled", "output_hash": "01a431ce7361dad4d4563ba12950bf58a8e80057b017e394a1ba62faf636ba50", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.002709, "p95_ms": 0.0032043999999999996, "repeats": 7, "samples_ms": [ 0.003, 0.003292, 0.003, 0.002666, 0.002709, 0.002666, 0.002625 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.008958, "p95_ms": 0.0095829, "repeats": 7, "samples_ms": [ 0.009708, 0.009041, 0.008958, 0.009291, 0.008708, 0.008458, 0.008083 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-13, "rtol": 1e-12 }, "variant": "ordinary", "warnings": [] }, { "artifacts": { "bundle": "observations/9cd278eec7e602790c8866135917bfb9d28ea58a55729ae99ab8480e57e1f768/reproducer.zip", "bundle_sha256": "0c088d38dd6b5825daa72d5be68e59fafd13865cd18447b85a79b22e7e2960a7", "case": "artifacts/4ad9551eec71170c8677e132d81e576fc9efb960117eb75e1066d5db1a816290/case.json", "inputs": "artifacts/4ad9551eec71170c8677e132d81e576fc9efb960117eb75e1066d5db1a816290/inputs.npz", "manifest": "artifacts/4ad9551eec71170c8677e132d81e576fc9efb960117eb75e1066d5db1a816290/manifest.json", "model": "artifacts/4ad9551eec71170c8677e132d81e576fc9efb960117eb75e1066d5db1a816290/model.onnx", "outputs": "observations/9cd278eec7e602790c8866135917bfb9d28ea58a55729ae99ab8480e57e1f768/outputs.npz" }, "candidate_group": "65954f4d7ca7f775bdc5ee6ae1a058219958ff9faed6c31dcb81095c5184c2cc", "case_id": "4ad9551eec71170c8677e132d81e576fc9efb960117eb75e1066d5db1a816290", "category": "normalization", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 46 ], "backend": "reference", "elements": 184, "expected_dtype": "float32", "expected_shape": [ 4, 46 ], "first_mismatch": null, "max_abs": 2.9802322387695312e-08, "max_budget_ratio": 0.008444448768002212, "max_rel": 1.6884345274054873e-07, "max_ulp": 2, "mismatched": 0, "name": "scaled_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 8.435292220276835e-08, "max_budget_ratio": 0.00766844747297894, "max_rel": 8.435292220276835e-08, "max_ulp": 379891789, "mismatched": 0, "name": "l2_unit_or_zero", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 46 ], "backend": "reference", "elements": 184, "expected_dtype": "float32", "expected_shape": [ 4, 46 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 46 ], "backend": "reference", "elements": 184, "expected_dtype": "float32", "expected_shape": [ 4, 46 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 46 ], "backend": "reference", "elements": 184, "expected_dtype": "float32", "expected_shape": [ 4, 46 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 46 ], "backend": "reference", "elements": 368, "expected_dtype": "float32", "expected_shape": [ 8, 46 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 46 ], "backend": "reference", "elements": 184, "expected_dtype": "float32", "expected_shape": [ 4, 46 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 46 ], "backend": "ort", "elements": 184, "expected_dtype": "float32", "expected_shape": [ 4, 46 ], "first_mismatch": null, "max_abs": 2.9802322387695312e-08, "max_budget_ratio": 0.01175560192931528, "max_rel": 1.9413180326411272e-07, "max_ulp": 2, "mismatched": 0, "name": "scaled_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 1.1680985245732245e-07, "max_budget_ratio": 0.01061907749612022, "max_rel": 1.1680985245732245e-07, "max_ulp": 526064808, "mismatched": 0, "name": "l2_unit_or_zero", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 46 ], "backend": "ort", "elements": 184, "expected_dtype": "float32", "expected_shape": [ 4, 46 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 46 ], "backend": "ort", "elements": 184, "expected_dtype": "float32", "expected_shape": [ 4, 46 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 46 ], "backend": "ort", "elements": 184, "expected_dtype": "float32", "expected_shape": [ 4, 46 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 46 ], "backend": "ort", "elements": 368, "expected_dtype": "float32", "expected_shape": [ 8, 46 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 46 ], "backend": "ort", "elements": 184, "expected_dtype": "float32", "expected_shape": [ 4, 46 ], "first_mismatch": null, "max_abs": 2.9802322387695312e-08, "max_budget_ratio": 0.010005757613103308, "max_rel": 1.7569504961628201e-07, "max_ulp": 2, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } } ], "comparison": { "actual_dtype": "float32", "actual_shape": [ 4, 46 ], "elements": 184, "expected_dtype": "float32", "expected_shape": [ 4, 46 ], "first_mismatch": null, "max_abs": 5.960464477539063e-08, "max_budget_ratio": 0.01483377105566653, "max_rel": 2.0969862428266034e-07, "max_ulp": 2, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, "comparison_key": "c845cd3288eb224dc51c89e197d53929d6852e7d94b4a5b212ea068a4bfac087", "contract": "unit_norm", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "float32", "shape": [ 4, 46 ] } }, "ir_version": 10, "name": "LpNormalization/random-004/float32/4x46/seed-20260906", "observation_id": "9cd278eec7e602790c8866135917bfb9d28ea58a55729ae99ab8480e57e1f768", "operation": "LpNormalization", "operators": [ "LpNormalization" ], "opset": 19, "optimization": "all", "output_hash": "5eaf1d51084bc4c20a61e4da338c6edf135c9de3b818fb0b7e67c2aa007959fa", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.003, "p95_ms": 0.0037124999999999997, "repeats": 7, "samples_ms": [ 0.003625, 0.00375, 0.003042, 0.003, 0.00275, 0.002834, 0.002708 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.009, "p95_ms": 0.0099126, "repeats": 7, "samples_ms": [ 0.010125, 0.009375, 0.008792, 0.008833, 0.009417, 0.009, 0.008625 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-06, "rtol": 1e-05 }, "variant": "random-004", "warnings": [] }, { "artifacts": { "bundle": "observations/1f1f09c4f6561320c28e44b4823bb68a0e7ecc70f3e642d568c4291dd42f6849/reproducer.zip", "bundle_sha256": "12c73569c4ee90d876e336e33f3d063f6d6a623b22e34615060f0f2b6cee4445", "case": "artifacts/4ad9551eec71170c8677e132d81e576fc9efb960117eb75e1066d5db1a816290/case.json", "inputs": "artifacts/4ad9551eec71170c8677e132d81e576fc9efb960117eb75e1066d5db1a816290/inputs.npz", "manifest": "artifacts/4ad9551eec71170c8677e132d81e576fc9efb960117eb75e1066d5db1a816290/manifest.json", "model": "artifacts/4ad9551eec71170c8677e132d81e576fc9efb960117eb75e1066d5db1a816290/model.onnx", "outputs": "observations/1f1f09c4f6561320c28e44b4823bb68a0e7ecc70f3e642d568c4291dd42f6849/outputs.npz" }, "candidate_group": "65954f4d7ca7f775bdc5ee6ae1a058219958ff9faed6c31dcb81095c5184c2cc", "case_id": "4ad9551eec71170c8677e132d81e576fc9efb960117eb75e1066d5db1a816290", "category": "normalization", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 46 ], "backend": "reference", "elements": 184, "expected_dtype": "float32", "expected_shape": [ 4, 46 ], "first_mismatch": null, "max_abs": 2.9802322387695312e-08, "max_budget_ratio": 0.008444448768002212, "max_rel": 1.6884345274054873e-07, "max_ulp": 2, "mismatched": 0, "name": "scaled_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 8.435292220276835e-08, "max_budget_ratio": 0.00766844747297894, "max_rel": 8.435292220276835e-08, "max_ulp": 379891789, "mismatched": 0, "name": "l2_unit_or_zero", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 46 ], "backend": "reference", "elements": 184, "expected_dtype": "float32", "expected_shape": [ 4, 46 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 46 ], "backend": "reference", "elements": 184, "expected_dtype": "float32", "expected_shape": [ 4, 46 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 46 ], "backend": "reference", "elements": 184, "expected_dtype": "float32", "expected_shape": [ 4, 46 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 46 ], "backend": "reference", "elements": 368, "expected_dtype": "float32", "expected_shape": [ 8, 46 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 46 ], "backend": "reference", "elements": 184, "expected_dtype": "float32", "expected_shape": [ 4, 46 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 46 ], "backend": "ort", "elements": 184, "expected_dtype": "float32", "expected_shape": [ 4, 46 ], "first_mismatch": null, "max_abs": 2.9802322387695312e-08, "max_budget_ratio": 0.01175560192931528, "max_rel": 1.9413180326411272e-07, "max_ulp": 2, "mismatched": 0, "name": "scaled_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 1.1680985245732245e-07, "max_budget_ratio": 0.01061907749612022, "max_rel": 1.1680985245732245e-07, "max_ulp": 526064808, "mismatched": 0, "name": "l2_unit_or_zero", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 46 ], "backend": "ort", "elements": 184, "expected_dtype": "float32", "expected_shape": [ 4, 46 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 46 ], "backend": "ort", "elements": 184, "expected_dtype": "float32", "expected_shape": [ 4, 46 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 46 ], "backend": "ort", "elements": 184, "expected_dtype": "float32", "expected_shape": [ 4, 46 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 46 ], "backend": "ort", "elements": 368, "expected_dtype": "float32", "expected_shape": [ 8, 46 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 46 ], "backend": "ort", "elements": 184, "expected_dtype": "float32", "expected_shape": [ 4, 46 ], "first_mismatch": null, "max_abs": 2.9802322387695312e-08, "max_budget_ratio": 0.010005757613103308, "max_rel": 1.7569504961628201e-07, "max_ulp": 2, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } } ], "comparison": { "actual_dtype": "float32", "actual_shape": [ 4, 46 ], "elements": 184, "expected_dtype": "float32", "expected_shape": [ 4, 46 ], "first_mismatch": null, "max_abs": 5.960464477539063e-08, "max_budget_ratio": 0.01483377105566653, "max_rel": 2.0969862428266034e-07, "max_ulp": 2, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, "comparison_key": "cc966d0d502836dd44fe025c9f212d5b29e9a82d639a131ca234c755a4bf82c6", "contract": "unit_norm", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "float32", "shape": [ 4, 46 ] } }, "ir_version": 10, "name": "LpNormalization/random-004/float32/4x46/seed-20260906", "observation_id": "1f1f09c4f6561320c28e44b4823bb68a0e7ecc70f3e642d568c4291dd42f6849", "operation": "LpNormalization", "operators": [ "LpNormalization" ], "opset": 19, "optimization": "disabled", "output_hash": "5eaf1d51084bc4c20a61e4da338c6edf135c9de3b818fb0b7e67c2aa007959fa", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.002958, "p95_ms": 0.0032917, "repeats": 7, "samples_ms": [ 0.003291, 0.003292, 0.003, 0.002958, 0.002708, 0.002958, 0.0025 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.009, "p95_ms": 0.009753699999999999, "repeats": 7, "samples_ms": [ 0.009916, 0.009375, 0.009167, 0.009, 0.008708, 0.009, 0.008791 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-06, "rtol": 1e-05 }, "variant": "random-004", "warnings": [] }, { "artifacts": { "bundle": "observations/e8628541aa50fdb79646e4cdb26442dd3eb124e9c0f6074d70591d992c5eeb92/reproducer.zip", "bundle_sha256": "d01eacb4e6708741ca404b58af0d12b086ba378600316257bde0704b6b2750e5", "case": "artifacts/844bb0ffccb62b59f6df410b3b516e981f174a3a067ce73aca091074ef2b86d4/case.json", "inputs": "artifacts/844bb0ffccb62b59f6df410b3b516e981f174a3a067ce73aca091074ef2b86d4/inputs.npz", "manifest": "artifacts/844bb0ffccb62b59f6df410b3b516e981f174a3a067ce73aca091074ef2b86d4/manifest.json", "model": "artifacts/844bb0ffccb62b59f6df410b3b516e981f174a3a067ce73aca091074ef2b86d4/model.onnx", "outputs": "observations/e8628541aa50fdb79646e4cdb26442dd3eb124e9c0f6074d70591d992c5eeb92/outputs.npz" }, "candidate_group": "cffa05fa142ad2f7bfb1139273c755d8f1c3ffee1e2cafb635f372d04672987e", "case_id": "844bb0ffccb62b59f6df410b3b516e981f174a3a067ce73aca091074ef2b86d4", "category": "normalization", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 108 ], "backend": "reference", "elements": 432, "expected_dtype": "float32", "expected_shape": [ 4, 108 ], "first_mismatch": null, "max_abs": 2.9802322387695312e-08, "max_budget_ratio": 0.009924815847771268, "max_rel": 1.7024365612551995e-07, "max_ulp": 2, "mismatched": 0, "name": "scaled_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 8.958203778774987e-08, "max_budget_ratio": 0.008143821617068168, "max_rel": 8.958203778774987e-08, "max_ulp": 403441632, "mismatched": 0, "name": "l2_unit_or_zero", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 108 ], "backend": "reference", "elements": 432, "expected_dtype": "float32", "expected_shape": [ 4, 108 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 108 ], "backend": "reference", "elements": 432, "expected_dtype": "float32", "expected_shape": [ 4, 108 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 108 ], "backend": "reference", "elements": 432, "expected_dtype": "float32", "expected_shape": [ 4, 108 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 108 ], "backend": "reference", "elements": 864, "expected_dtype": "float32", "expected_shape": [ 8, 108 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 108 ], "backend": "reference", "elements": 432, "expected_dtype": "float32", "expected_shape": [ 4, 108 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 108 ], "backend": "ort", "elements": 432, "expected_dtype": "float32", "expected_shape": [ 4, 108 ], "first_mismatch": null, "max_abs": 2.9802322387695312e-08, "max_budget_ratio": 0.013001245636343172, "max_rel": 2.3822858938420054e-07, "max_ulp": 3, "mismatched": 0, "name": "scaled_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 1.3925480935039758e-07, "max_budget_ratio": 0.012659528122763416, "max_rel": 1.3925480935039758e-07, "max_ulp": 1254295815, "mismatched": 0, "name": "l2_unit_or_zero", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 108 ], "backend": "ort", "elements": 432, "expected_dtype": "float32", "expected_shape": [ 4, 108 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 108 ], "backend": "ort", "elements": 432, "expected_dtype": "float32", "expected_shape": [ 4, 108 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 108 ], "backend": "ort", "elements": 432, "expected_dtype": "float32", "expected_shape": [ 4, 108 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 108 ], "backend": "ort", "elements": 864, "expected_dtype": "float32", "expected_shape": [ 8, 108 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 108 ], "backend": "ort", "elements": 432, "expected_dtype": "float32", "expected_shape": [ 4, 108 ], "first_mismatch": null, "max_abs": 5.960464477539063e-08, "max_budget_ratio": 0.019028836317584504, "max_rel": 3.0651090463628396e-07, "max_ulp": 4, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } } ], "comparison": { "actual_dtype": "float32", "actual_shape": [ 4, 108 ], "elements": 432, "expected_dtype": "float32", "expected_shape": [ 4, 108 ], "first_mismatch": null, "max_abs": 2.9802322387695312e-08, "max_budget_ratio": 0.013001245636343172, "max_rel": 2.3822858938420054e-07, "max_ulp": 3, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, "comparison_key": "01cf136a5f3db19419a359c528e4965d2df0d60a94a83e86c108d4bf2819cc86", "contract": "unit_norm", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "float32", "shape": [ 4, 108 ] } }, "ir_version": 10, "name": "LpNormalization/random-011/float32/4x108/seed-20260906", "observation_id": "e8628541aa50fdb79646e4cdb26442dd3eb124e9c0f6074d70591d992c5eeb92", "operation": "LpNormalization", "operators": [ "LpNormalization" ], "opset": 19, "optimization": "all", "output_hash": "52ecff92f3acc3d49c5cba96a634742e0e2a6a65ae72001dddae79d3e55bae5a", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.002791, "p95_ms": 0.0033580999999999997, "repeats": 7, "samples_ms": [ 0.002791, 0.003458, 0.002875, 0.003125, 0.002708, 0.002625, 0.002625 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.009, "p95_ms": 0.010367, "repeats": 7, "samples_ms": [ 0.010667, 0.009625, 0.009, 0.008708, 0.008625, 0.00875, 0.009667 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-06, "rtol": 1e-05 }, "variant": "random-011", "warnings": [] }, { "artifacts": { "bundle": "observations/52bd084ceda8f28d952a3820af410ba40487cde6596f5f63b27400c13710349a/reproducer.zip", "bundle_sha256": "24db08d02c0f5bd3275a32d953bba038c2003d478f56c3b597489907e06c290e", "case": "artifacts/844bb0ffccb62b59f6df410b3b516e981f174a3a067ce73aca091074ef2b86d4/case.json", "inputs": "artifacts/844bb0ffccb62b59f6df410b3b516e981f174a3a067ce73aca091074ef2b86d4/inputs.npz", "manifest": "artifacts/844bb0ffccb62b59f6df410b3b516e981f174a3a067ce73aca091074ef2b86d4/manifest.json", "model": "artifacts/844bb0ffccb62b59f6df410b3b516e981f174a3a067ce73aca091074ef2b86d4/model.onnx", "outputs": "observations/52bd084ceda8f28d952a3820af410ba40487cde6596f5f63b27400c13710349a/outputs.npz" }, "candidate_group": "cffa05fa142ad2f7bfb1139273c755d8f1c3ffee1e2cafb635f372d04672987e", "case_id": "844bb0ffccb62b59f6df410b3b516e981f174a3a067ce73aca091074ef2b86d4", "category": "normalization", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 108 ], "backend": "reference", "elements": 432, "expected_dtype": "float32", "expected_shape": [ 4, 108 ], "first_mismatch": null, "max_abs": 2.9802322387695312e-08, "max_budget_ratio": 0.009924815847771268, "max_rel": 1.7024365612551995e-07, "max_ulp": 2, "mismatched": 0, "name": "scaled_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 8.958203778774987e-08, "max_budget_ratio": 0.008143821617068168, "max_rel": 8.958203778774987e-08, "max_ulp": 403441632, "mismatched": 0, "name": "l2_unit_or_zero", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 108 ], "backend": "reference", "elements": 432, "expected_dtype": "float32", "expected_shape": [ 4, 108 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 108 ], "backend": "reference", "elements": 432, "expected_dtype": "float32", "expected_shape": [ 4, 108 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 108 ], "backend": "reference", "elements": 432, "expected_dtype": "float32", "expected_shape": [ 4, 108 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 108 ], "backend": "reference", "elements": 864, "expected_dtype": "float32", "expected_shape": [ 8, 108 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 108 ], "backend": "reference", "elements": 432, "expected_dtype": "float32", "expected_shape": [ 4, 108 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 108 ], "backend": "ort", "elements": 432, "expected_dtype": "float32", "expected_shape": [ 4, 108 ], "first_mismatch": null, "max_abs": 2.9802322387695312e-08, "max_budget_ratio": 0.013001245636343172, "max_rel": 2.3822858938420054e-07, "max_ulp": 3, "mismatched": 0, "name": "scaled_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 1.3925480935039758e-07, "max_budget_ratio": 0.012659528122763416, "max_rel": 1.3925480935039758e-07, "max_ulp": 1254295815, "mismatched": 0, "name": "l2_unit_or_zero", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 108 ], "backend": "ort", "elements": 432, "expected_dtype": "float32", "expected_shape": [ 4, 108 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 108 ], "backend": "ort", "elements": 432, "expected_dtype": "float32", "expected_shape": [ 4, 108 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 108 ], "backend": "ort", "elements": 432, "expected_dtype": "float32", "expected_shape": [ 4, 108 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 108 ], "backend": "ort", "elements": 864, "expected_dtype": "float32", "expected_shape": [ 8, 108 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 108 ], "backend": "ort", "elements": 432, "expected_dtype": "float32", "expected_shape": [ 4, 108 ], "first_mismatch": null, "max_abs": 5.960464477539063e-08, "max_budget_ratio": 0.019028836317584504, "max_rel": 3.0651090463628396e-07, "max_ulp": 4, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } } ], "comparison": { "actual_dtype": "float32", "actual_shape": [ 4, 108 ], "elements": 432, "expected_dtype": "float32", "expected_shape": [ 4, 108 ], "first_mismatch": null, "max_abs": 2.9802322387695312e-08, "max_budget_ratio": 0.013001245636343172, "max_rel": 2.3822858938420054e-07, "max_ulp": 3, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, "comparison_key": "0494ddfd315fe35d16dc84ddab211499a1aac0405105bf02c8cb35ad22d5fc61", "contract": "unit_norm", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "float32", "shape": [ 4, 108 ] } }, "ir_version": 10, "name": "LpNormalization/random-011/float32/4x108/seed-20260906", "observation_id": "52bd084ceda8f28d952a3820af410ba40487cde6596f5f63b27400c13710349a", "operation": "LpNormalization", "operators": [ "LpNormalization" ], "opset": 19, "optimization": "disabled", "output_hash": "52ecff92f3acc3d49c5cba96a634742e0e2a6a65ae72001dddae79d3e55bae5a", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.002917, "p95_ms": 0.0036205999999999994, "repeats": 7, "samples_ms": [ 0.003125, 0.003833, 0.003083, 0.002917, 0.002834, 0.002625, 0.002792 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.009292, "p95_ms": 0.0101249, "repeats": 7, "samples_ms": [ 0.01025, 0.009833, 0.009292, 0.009041, 0.008833, 0.009292, 0.008792 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-06, "rtol": 1e-05 }, "variant": "random-011", "warnings": [] }, { "artifacts": { "bundle": "observations/232c20ff5f5d8498b485b4e26914dbe71de6f8e1dd10d110662e37aa0e5d4d42/reproducer.zip", "bundle_sha256": "3e179a6f1c6473457fd7c54071ca22c2946ce76b1ba187904c039e9cd838a0d9", "case": "artifacts/dcf6a1b45b5c1366664c2b6e71a7194db9a0e8745bcbffc4b613def02fbc11db/case.json", "inputs": "artifacts/dcf6a1b45b5c1366664c2b6e71a7194db9a0e8745bcbffc4b613def02fbc11db/inputs.npz", "manifest": "artifacts/dcf6a1b45b5c1366664c2b6e71a7194db9a0e8745bcbffc4b613def02fbc11db/manifest.json", "model": "artifacts/dcf6a1b45b5c1366664c2b6e71a7194db9a0e8745bcbffc4b613def02fbc11db/model.onnx", "outputs": "observations/232c20ff5f5d8498b485b4e26914dbe71de6f8e1dd10d110662e37aa0e5d4d42/outputs.npz" }, "candidate_group": "8336aff250f077bbc47a1c7a254d9986c5bb2709617a46546d124facbbd61905", "case_id": "dcf6a1b45b5c1366664c2b6e71a7194db9a0e8745bcbffc4b613def02fbc11db", "category": "normalization", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 64 ], "backend": "reference", "elements": 256, "expected_dtype": "float32", "expected_shape": [ 4, 64 ], "first_mismatch": null, "max_abs": 1.4901161193847656e-08, "max_budget_ratio": 0.006530203097021957, "max_rel": 1.1850393990048988e-07, "max_ulp": 1, "mismatched": 0, "name": "scaled_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 6.049397338614426e-08, "max_budget_ratio": 0.005499452126013114, "max_rel": 6.049397338614426e-08, "max_ulp": 272440636, "mismatched": 0, "name": "l2_unit_or_zero", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 64 ], "backend": "reference", "elements": 256, "expected_dtype": "float32", "expected_shape": [ 4, 64 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 64 ], "backend": "reference", "elements": 256, "expected_dtype": "float32", "expected_shape": [ 4, 64 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 64 ], "backend": "reference", "elements": 256, "expected_dtype": "float32", "expected_shape": [ 4, 64 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 64 ], "backend": "reference", "elements": 512, "expected_dtype": "float32", "expected_shape": [ 8, 64 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 64 ], "backend": "reference", "elements": 256, "expected_dtype": "float32", "expected_shape": [ 4, 64 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 64 ], "backend": "ort", "elements": 256, "expected_dtype": "float32", "expected_shape": [ 4, 64 ], "first_mismatch": null, "max_abs": 5.960464477539063e-08, "max_budget_ratio": 0.014827270600772536, "max_rel": 2.2931233015838946e-07, "max_ulp": 3, "mismatched": 0, "name": "scaled_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 1.594529145387824e-07, "max_budget_ratio": 0.014495719503525671, "max_rel": 1.594529145387824e-07, "max_ulp": 1436224173, "mismatched": 0, "name": "l2_unit_or_zero", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 64 ], "backend": "ort", "elements": 256, "expected_dtype": "float32", "expected_shape": [ 4, 64 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 64 ], "backend": "ort", "elements": 256, "expected_dtype": "float32", "expected_shape": [ 4, 64 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 64 ], "backend": "ort", "elements": 256, "expected_dtype": "float32", "expected_shape": [ 4, 64 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 64 ], "backend": "ort", "elements": 512, "expected_dtype": "float32", "expected_shape": [ 8, 64 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 64 ], "backend": "ort", "elements": 256, "expected_dtype": "float32", "expected_shape": [ 4, 64 ], "first_mismatch": null, "max_abs": 2.9802322387695312e-08, "max_budget_ratio": 0.011259326127980121, "max_rel": 1.8096000367710727e-07, "max_ulp": 2, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } } ], "comparison": { "actual_dtype": "float32", "actual_shape": [ 4, 64 ], "elements": 256, "expected_dtype": "float32", "expected_shape": [ 4, 64 ], "first_mismatch": null, "max_abs": 5.960464477539063e-08, "max_budget_ratio": 0.017864121508110214, "max_rel": 3.270281934275836e-07, "max_ulp": 4, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, "comparison_key": "3f9194b7522beefe1e48ff3c8512495352283f76a376d90a36f8cb1a2eebeecb", "contract": "unit_norm", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "float32", "shape": [ 4, 64 ] } }, "ir_version": 10, "name": "LpNormalization/random-018/float32/4x64/seed-20260906", "observation_id": "232c20ff5f5d8498b485b4e26914dbe71de6f8e1dd10d110662e37aa0e5d4d42", "operation": "LpNormalization", "operators": [ "LpNormalization" ], "opset": 19, "optimization": "all", "output_hash": "fd86b0cb382ce1cd1ad7f24ea661798471be02a45da776a4b346ab627b126ba2", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.002917, "p95_ms": 0.00325, "repeats": 7, "samples_ms": [ 0.00325, 0.00325, 0.002917, 0.002958, 0.00275, 0.002708, 0.002875 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.009292, "p95_ms": 0.060312399999999954, "repeats": 7, "samples_ms": [ 0.010292, 0.013208, 0.009292, 0.008958, 0.008625, 0.00875, 0.0805 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-06, "rtol": 1e-05 }, "variant": "random-018", "warnings": [] }, { "artifacts": { "bundle": "observations/abc79f8a1e16bf10e90d49a735ebe2f064426e38497d16e5768c3ba8b43a92e4/reproducer.zip", "bundle_sha256": "01a2079a52c23cf5ca91889b1ead7d64210a86a22b5b921658ebae3f9aa6fa9d", "case": "artifacts/dcf6a1b45b5c1366664c2b6e71a7194db9a0e8745bcbffc4b613def02fbc11db/case.json", "inputs": "artifacts/dcf6a1b45b5c1366664c2b6e71a7194db9a0e8745bcbffc4b613def02fbc11db/inputs.npz", "manifest": "artifacts/dcf6a1b45b5c1366664c2b6e71a7194db9a0e8745bcbffc4b613def02fbc11db/manifest.json", "model": "artifacts/dcf6a1b45b5c1366664c2b6e71a7194db9a0e8745bcbffc4b613def02fbc11db/model.onnx", "outputs": "observations/abc79f8a1e16bf10e90d49a735ebe2f064426e38497d16e5768c3ba8b43a92e4/outputs.npz" }, "candidate_group": "8336aff250f077bbc47a1c7a254d9986c5bb2709617a46546d124facbbd61905", "case_id": "dcf6a1b45b5c1366664c2b6e71a7194db9a0e8745bcbffc4b613def02fbc11db", "category": "normalization", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 64 ], "backend": "reference", "elements": 256, "expected_dtype": "float32", "expected_shape": [ 4, 64 ], "first_mismatch": null, "max_abs": 1.4901161193847656e-08, "max_budget_ratio": 0.006530203097021957, "max_rel": 1.1850393990048988e-07, "max_ulp": 1, "mismatched": 0, "name": "scaled_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 6.049397338614426e-08, "max_budget_ratio": 0.005499452126013114, "max_rel": 6.049397338614426e-08, "max_ulp": 272440636, "mismatched": 0, "name": "l2_unit_or_zero", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 64 ], "backend": "reference", "elements": 256, "expected_dtype": "float32", "expected_shape": [ 4, 64 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 64 ], "backend": "reference", "elements": 256, "expected_dtype": "float32", "expected_shape": [ 4, 64 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 64 ], "backend": "reference", "elements": 256, "expected_dtype": "float32", "expected_shape": [ 4, 64 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 64 ], "backend": "reference", "elements": 512, "expected_dtype": "float32", "expected_shape": [ 8, 64 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 64 ], "backend": "reference", "elements": 256, "expected_dtype": "float32", "expected_shape": [ 4, 64 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 64 ], "backend": "ort", "elements": 256, "expected_dtype": "float32", "expected_shape": [ 4, 64 ], "first_mismatch": null, "max_abs": 5.960464477539063e-08, "max_budget_ratio": 0.014827270600772536, "max_rel": 2.2931233015838946e-07, "max_ulp": 3, "mismatched": 0, "name": "scaled_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 1.594529145387824e-07, "max_budget_ratio": 0.014495719503525671, "max_rel": 1.594529145387824e-07, "max_ulp": 1436224173, "mismatched": 0, "name": "l2_unit_or_zero", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 64 ], "backend": "ort", "elements": 256, "expected_dtype": "float32", "expected_shape": [ 4, 64 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 64 ], "backend": "ort", "elements": 256, "expected_dtype": "float32", "expected_shape": [ 4, 64 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 64 ], "backend": "ort", "elements": 256, "expected_dtype": "float32", "expected_shape": [ 4, 64 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 64 ], "backend": "ort", "elements": 512, "expected_dtype": "float32", "expected_shape": [ 8, 64 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 64 ], "backend": "ort", "elements": 256, "expected_dtype": "float32", "expected_shape": [ 4, 64 ], "first_mismatch": null, "max_abs": 2.9802322387695312e-08, "max_budget_ratio": 0.011259326127980121, "max_rel": 1.8096000367710727e-07, "max_ulp": 2, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } } ], "comparison": { "actual_dtype": "float32", "actual_shape": [ 4, 64 ], "elements": 256, "expected_dtype": "float32", "expected_shape": [ 4, 64 ], "first_mismatch": null, "max_abs": 5.960464477539063e-08, "max_budget_ratio": 0.017864121508110214, "max_rel": 3.270281934275836e-07, "max_ulp": 4, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, "comparison_key": "ca81fb192c391d1141b7485c57685e035f95d4e9325c3df47169ce39c5860d5d", "contract": "unit_norm", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "float32", "shape": [ 4, 64 ] } }, "ir_version": 10, "name": "LpNormalization/random-018/float32/4x64/seed-20260906", "observation_id": "abc79f8a1e16bf10e90d49a735ebe2f064426e38497d16e5768c3ba8b43a92e4", "operation": "LpNormalization", "operators": [ "LpNormalization" ], "opset": 19, "optimization": "disabled", "output_hash": "fd86b0cb382ce1cd1ad7f24ea661798471be02a45da776a4b346ab627b126ba2", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.003042, "p95_ms": 0.0039752, "repeats": 7, "samples_ms": [ 0.003042, 0.00425, 0.003334, 0.003125, 0.002959, 0.002875, 0.002709 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.008667, "p95_ms": 0.009933999999999998, "repeats": 7, "samples_ms": [ 0.010084, 0.009584, 0.009041, 0.008666, 0.008541, 0.008667, 0.008542 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-06, "rtol": 1e-05 }, "variant": "random-018", "warnings": [] }, { "artifacts": { "bundle": "observations/a14a3aae1ab076aa1f49f7ebec147cfb0adc9b55291f7bc1e5c4fd227a9c8481/reproducer.zip", "bundle_sha256": "709be1e5e80a4c216c37d0f3cce8fe2791d6d189ba36ed6dddfd0250913c0e64", "case": "artifacts/554a9eabfdd5be921eab3dc6b654d994d0ea1aa6e2fd79dc5b497574d47f5620/case.json", "inputs": "artifacts/554a9eabfdd5be921eab3dc6b654d994d0ea1aa6e2fd79dc5b497574d47f5620/inputs.npz", "manifest": "artifacts/554a9eabfdd5be921eab3dc6b654d994d0ea1aa6e2fd79dc5b497574d47f5620/manifest.json", "model": "artifacts/554a9eabfdd5be921eab3dc6b654d994d0ea1aa6e2fd79dc5b497574d47f5620/model.onnx", "outputs": "observations/a14a3aae1ab076aa1f49f7ebec147cfb0adc9b55291f7bc1e5c4fd227a9c8481/outputs.npz" }, "candidate_group": "742f7302ee813784abcf07f3bdb4f93ee015b1d9e774016cb044fe92dae70cbf", "case_id": "554a9eabfdd5be921eab3dc6b654d994d0ea1aa6e2fd79dc5b497574d47f5620", "category": "normalization", "checks": [ { "backend": "reference", "name": "finite_output", "passed": false }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": [ 0, 0 ], "max_abs": 0.23779296875, "max_budget_ratio": 69.33371298405467, "max_rel": 0.4069223573433115, "max_ulp": 580, "mismatched": 67, "name": "scaled_float64_oracle", "nonfinite": 17, "numerical_divergence": true, "passed": false, "reason": "nonfinite_output", "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": [ 0 ], "max_abs": 0.4063544976297362, "max_budget_ratio": 73.8826359326793, "max_rel": 0.4063544976297362, "max_ulp": 1830057964105605, "mismatched": 4, "name": "l2_unit_or_zero", "nonfinite": 1, "numerical_divergence": true, "passed": false, "reason": "nonfinite_output", "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": [ 3, 0 ], "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 17, "name": "batch_partition", "nonfinite": 17, "numerical_divergence": false, "passed": false, "reason": "nonfinite_output", "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": [ 0, 0 ], "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 17, "name": "batch_permutation", "nonfinite": 17, "numerical_divergence": false, "passed": false, "reason": "nonfinite_output", "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": [ 3, 0 ], "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 17, "name": "batch_reshape", "nonfinite": 17, "numerical_divergence": false, "passed": false, "reason": "nonfinite_output", "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 8, 17 ], "backend": "reference", "elements": 136, "expected_dtype": "float16", "expected_shape": [ 8, 17 ], "first_mismatch": [ 3, 0 ], "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 34, "name": "batch_duplicate", "nonfinite": 34, "numerical_divergence": false, "passed": false, "reason": "nonfinite_output", "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": [ 3, 0 ], "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 17, "name": "equivalent_graph", "nonfinite": 17, "numerical_divergence": true, "passed": false, "reason": "nonfinite_output", "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "scaled_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 9.328367751582078e-05, "max_budget_ratio": 0.016960668639240145, "max_rel": 9.328367751582078e-05, "max_ulp": 420112335300, "mismatched": 0, "name": "l2_unit_or_zero", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 8, 17 ], "backend": "ort", "elements": 136, "expected_dtype": "float16", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } } ], "comparison": { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": [ 0, 0 ], "max_abs": 0.23779296875, "max_budget_ratio": 51.48535786023893, "max_rel": 0.2892287234042553, "max_ulp": 580, "mismatched": 67, "nonfinite": 17, "numerical_divergence": true, "passed": false, "reason": "nonfinite_output", "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, "comparison_key": "fee2aac9e032a44236b838d892eda1b5f0db450029a2cbdd7892975570b05718", "contract": "unit_norm", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "float16", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "LpNormalization/tiny/float16/4x17/seed-20260906", "observation_id": "a14a3aae1ab076aa1f49f7ebec147cfb0adc9b55291f7bc1e5c4fd227a9c8481", "operation": "LpNormalization", "operators": [ "LpNormalization" ], "opset": 19, "optimization": "all", "output_hash": "b4559a4135ad44e62d34c5efa3f528269bdf681005ea80b13f0663c310188468", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "divergence", "timings": { "ort": { "median_ms": 0.009375, "p95_ms": 0.01005, "repeats": 7, "samples_ms": [ 0.010125, 0.009875, 0.009459, 0.009083, 0.008625, 0.009375, 0.008708 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.011375, "p95_ms": 0.012787499999999999, "repeats": 7, "samples_ms": [ 0.013125, 0.012, 0.011625, 0.010958, 0.01075, 0.01075, 0.011375 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 0.0005, "rtol": 0.005 }, "variant": "tiny", "warnings": [ { "backend": "reference", "message": "divide by zero encountered in divide" }, { "backend": "reference", "message": "underflow encountered in multiply" }, { "backend": "reference", "message": "underflow encountered in power" } ] }, { "artifacts": { "bundle": "observations/99e1c8d4161bcbd8c92b279993f76e8609dfcda7200ce80ce0772de1154994e8/reproducer.zip", "bundle_sha256": "1efeb2a488eed7d246dbbe9ec4de336bd22e53619b8916918c39c752455823d6", "case": "artifacts/554a9eabfdd5be921eab3dc6b654d994d0ea1aa6e2fd79dc5b497574d47f5620/case.json", "inputs": "artifacts/554a9eabfdd5be921eab3dc6b654d994d0ea1aa6e2fd79dc5b497574d47f5620/inputs.npz", "manifest": "artifacts/554a9eabfdd5be921eab3dc6b654d994d0ea1aa6e2fd79dc5b497574d47f5620/manifest.json", "model": "artifacts/554a9eabfdd5be921eab3dc6b654d994d0ea1aa6e2fd79dc5b497574d47f5620/model.onnx", "outputs": "observations/99e1c8d4161bcbd8c92b279993f76e8609dfcda7200ce80ce0772de1154994e8/outputs.npz" }, "candidate_group": "742f7302ee813784abcf07f3bdb4f93ee015b1d9e774016cb044fe92dae70cbf", "case_id": "554a9eabfdd5be921eab3dc6b654d994d0ea1aa6e2fd79dc5b497574d47f5620", "category": "normalization", "checks": [ { "backend": "reference", "name": "finite_output", "passed": false }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": [ 0, 0 ], "max_abs": 0.23779296875, "max_budget_ratio": 69.33371298405467, "max_rel": 0.4069223573433115, "max_ulp": 580, "mismatched": 67, "name": "scaled_float64_oracle", "nonfinite": 17, "numerical_divergence": true, "passed": false, "reason": "nonfinite_output", "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": [ 0 ], "max_abs": 0.4063544976297362, "max_budget_ratio": 73.8826359326793, "max_rel": 0.4063544976297362, "max_ulp": 1830057964105605, "mismatched": 4, "name": "l2_unit_or_zero", "nonfinite": 1, "numerical_divergence": true, "passed": false, "reason": "nonfinite_output", "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": [ 3, 0 ], "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 17, "name": "batch_partition", "nonfinite": 17, "numerical_divergence": false, "passed": false, "reason": "nonfinite_output", "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": [ 0, 0 ], "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 17, "name": "batch_permutation", "nonfinite": 17, "numerical_divergence": false, "passed": false, "reason": "nonfinite_output", "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": [ 3, 0 ], "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 17, "name": "batch_reshape", "nonfinite": 17, "numerical_divergence": false, "passed": false, "reason": "nonfinite_output", "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 8, 17 ], "backend": "reference", "elements": 136, "expected_dtype": "float16", "expected_shape": [ 8, 17 ], "first_mismatch": [ 3, 0 ], "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 34, "name": "batch_duplicate", "nonfinite": 34, "numerical_divergence": false, "passed": false, "reason": "nonfinite_output", "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": [ 3, 0 ], "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 17, "name": "equivalent_graph", "nonfinite": 17, "numerical_divergence": true, "passed": false, "reason": "nonfinite_output", "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "scaled_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 9.328367751582078e-05, "max_budget_ratio": 0.016960668639240145, "max_rel": 9.328367751582078e-05, "max_ulp": 420112335300, "mismatched": 0, "name": "l2_unit_or_zero", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 8, 17 ], "backend": "ort", "elements": 136, "expected_dtype": "float16", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } } ], "comparison": { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": [ 0, 0 ], "max_abs": 0.23779296875, "max_budget_ratio": 51.48535786023893, "max_rel": 0.2892287234042553, "max_ulp": 580, "mismatched": 67, "nonfinite": 17, "numerical_divergence": true, "passed": false, "reason": "nonfinite_output", "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, "comparison_key": "71a0f9813deba68f23a709c16003214445966d5e4b3a68c356b46ff81e0342d3", "contract": "unit_norm", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "float16", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "LpNormalization/tiny/float16/4x17/seed-20260906", "observation_id": "99e1c8d4161bcbd8c92b279993f76e8609dfcda7200ce80ce0772de1154994e8", "operation": "LpNormalization", "operators": [ "LpNormalization" ], "opset": 19, "optimization": "disabled", "output_hash": "b4559a4135ad44e62d34c5efa3f528269bdf681005ea80b13f0663c310188468", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "divergence", "timings": { "ort": { "median_ms": 0.003709, "p95_ms": 0.0040213, "repeats": 7, "samples_ms": [ 0.003875, 0.004084, 0.003833, 0.003709, 0.003583, 0.003542, 0.003292 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.010834, "p95_ms": 0.0126544, "repeats": 7, "samples_ms": [ 0.013042, 0.01175, 0.011167, 0.010708, 0.0105, 0.010834, 0.010542 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 0.0005, "rtol": 0.005 }, "variant": "tiny", "warnings": [ { "backend": "reference", "message": "divide by zero encountered in divide" }, { "backend": "reference", "message": "underflow encountered in multiply" }, { "backend": "reference", "message": "underflow encountered in power" } ] }, { "artifacts": { "bundle": "observations/defdf560b6ffc0d0ab7d9b81d23fe74ddcf5d60ee19cf3bee0fb8e761c8d4c9d/reproducer.zip", "bundle_sha256": "17ba61284a8b01793ee5e76337018edfe568092e5483fdaf4a8b405d7a4ee0ae", "case": "artifacts/4c4c8a3ddaeb32674513f478e22ea94962f5d631e3add484e646476fbbe5e065/case.json", "inputs": "artifacts/4c4c8a3ddaeb32674513f478e22ea94962f5d631e3add484e646476fbbe5e065/inputs.npz", "manifest": "artifacts/4c4c8a3ddaeb32674513f478e22ea94962f5d631e3add484e646476fbbe5e065/manifest.json", "model": "artifacts/4c4c8a3ddaeb32674513f478e22ea94962f5d631e3add484e646476fbbe5e065/model.onnx", "outputs": "observations/defdf560b6ffc0d0ab7d9b81d23fe74ddcf5d60ee19cf3bee0fb8e761c8d4c9d/outputs.npz" }, "candidate_group": "afe2d3aa9e11fc6a451c88fe96b8f3fe756f3a6423b0abec0a9e06edcab37cc9", "case_id": "4c4c8a3ddaeb32674513f478e22ea94962f5d631e3add484e646476fbbe5e065", "category": "normalization", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 2.9802322387695312e-08, "max_budget_ratio": 0.00700667824721105, "max_rel": 1.1791555948869456e-07, "max_ulp": 1, "mismatched": 0, "name": "scaled_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 4.6999462766095235e-08, "max_budget_ratio": 0.004272678433281384, "max_rel": 4.6999462766095235e-08, "max_ulp": 211666763, "mismatched": 0, "name": "l2_unit_or_zero", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 17 ], "backend": "reference", "elements": 136, "expected_dtype": "float32", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 5.960464477539063e-08, "max_budget_ratio": 0.009379124821857591, "max_rel": 1.3557119806165724e-07, "max_ulp": 2, "mismatched": 0, "name": "scaled_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 1.0088856750023467e-07, "max_budget_ratio": 0.009171687954566788, "max_rel": 1.0088856750023467e-07, "max_ulp": 454361715, "mismatched": 0, "name": "l2_unit_or_zero", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 17 ], "backend": "ort", "elements": 136, "expected_dtype": "float32", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 5.960464477539063e-08, "max_budget_ratio": 0.009379123942177851, "max_rel": 1.2152777672284917e-07, "max_ulp": 2, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } } ], "comparison": { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 5.960464477539063e-08, "max_budget_ratio": 0.010333821069020149, "max_rel": 1.258329670545384e-07, "max_ulp": 2, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, "comparison_key": "b960ecf9f1425553bc5d6f8e4ac36837b9779bbff5eb4bd2b783ea4fc167e7c2", "contract": "unit_norm", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "float32", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "LpNormalization/tiny/float32/4x17/seed-20260906", "observation_id": "defdf560b6ffc0d0ab7d9b81d23fe74ddcf5d60ee19cf3bee0fb8e761c8d4c9d", "operation": "LpNormalization", "operators": [ "LpNormalization" ], "opset": 19, "optimization": "all", "output_hash": "4902202d89ed9aa082f5eecf3acbb52049ec11defa6a8edc86692568f0c1fcbd", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.003, "p95_ms": 0.0035999999999999995, "repeats": 7, "samples_ms": [ 0.003125, 0.00375, 0.00325, 0.003, 0.002708, 0.002791, 0.0025 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.008958, "p95_ms": 0.010279499999999999, "repeats": 7, "samples_ms": [ 0.010542, 0.009667, 0.009042, 0.008583, 0.008584, 0.008958, 0.008416 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-06, "rtol": 1e-05 }, "variant": "tiny", "warnings": [] }, { "artifacts": { "bundle": "observations/90721de81a7b0bd2fb1b5c62da322f9300caab865b35e1e23d977c2844e5ae4d/reproducer.zip", "bundle_sha256": "976382a2f2dbee619e35a1ede3d121630969fb5d97930bd856f7a4d90d687589", "case": "artifacts/4c4c8a3ddaeb32674513f478e22ea94962f5d631e3add484e646476fbbe5e065/case.json", "inputs": "artifacts/4c4c8a3ddaeb32674513f478e22ea94962f5d631e3add484e646476fbbe5e065/inputs.npz", "manifest": "artifacts/4c4c8a3ddaeb32674513f478e22ea94962f5d631e3add484e646476fbbe5e065/manifest.json", "model": "artifacts/4c4c8a3ddaeb32674513f478e22ea94962f5d631e3add484e646476fbbe5e065/model.onnx", "outputs": "observations/90721de81a7b0bd2fb1b5c62da322f9300caab865b35e1e23d977c2844e5ae4d/outputs.npz" }, "candidate_group": "afe2d3aa9e11fc6a451c88fe96b8f3fe756f3a6423b0abec0a9e06edcab37cc9", "case_id": "4c4c8a3ddaeb32674513f478e22ea94962f5d631e3add484e646476fbbe5e065", "category": "normalization", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 2.9802322387695312e-08, "max_budget_ratio": 0.00700667824721105, "max_rel": 1.1791555948869456e-07, "max_ulp": 1, "mismatched": 0, "name": "scaled_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 4.6999462766095235e-08, "max_budget_ratio": 0.004272678433281384, "max_rel": 4.6999462766095235e-08, "max_ulp": 211666763, "mismatched": 0, "name": "l2_unit_or_zero", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 17 ], "backend": "reference", "elements": 136, "expected_dtype": "float32", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 5.960464477539063e-08, "max_budget_ratio": 0.009379124821857591, "max_rel": 1.3557119806165724e-07, "max_ulp": 2, "mismatched": 0, "name": "scaled_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 1.0088856750023467e-07, "max_budget_ratio": 0.009171687954566788, "max_rel": 1.0088856750023467e-07, "max_ulp": 454361715, "mismatched": 0, "name": "l2_unit_or_zero", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 17 ], "backend": "ort", "elements": 136, "expected_dtype": "float32", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 5.960464477539063e-08, "max_budget_ratio": 0.009379123942177851, "max_rel": 1.2152777672284917e-07, "max_ulp": 2, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } } ], "comparison": { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 5.960464477539063e-08, "max_budget_ratio": 0.010333821069020149, "max_rel": 1.258329670545384e-07, "max_ulp": 2, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, "comparison_key": "be330f67c9cdaf63e42c9a8086f97f2193f6c96174cbfce5d5cbc78e11b90d58", "contract": "unit_norm", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "float32", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "LpNormalization/tiny/float32/4x17/seed-20260906", "observation_id": "90721de81a7b0bd2fb1b5c62da322f9300caab865b35e1e23d977c2844e5ae4d", "operation": "LpNormalization", "operators": [ "LpNormalization" ], "opset": 19, "optimization": "disabled", "output_hash": "4902202d89ed9aa082f5eecf3acbb52049ec11defa6a8edc86692568f0c1fcbd", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.002667, "p95_ms": 0.0031831, "repeats": 7, "samples_ms": [ 0.002833, 0.003208, 0.003125, 0.002667, 0.002583, 0.002417, 0.002583 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.008667, "p95_ms": 0.0099752, "repeats": 7, "samples_ms": [ 0.01025, 0.009334, 0.008667, 0.008458, 0.008167, 0.008708, 0.008125 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-06, "rtol": 1e-05 }, "variant": "tiny", "warnings": [] }, { "artifacts": { "bundle": "observations/02827c31df4a37addf4daed75bb32ba2cd624bf9e82ccec7bb72c30a281a61d3/reproducer.zip", "bundle_sha256": "9a630417ebd3ad467eabc157ecd3d46cf10684a3212987770418aab9fc5e9276", "case": "artifacts/b2054ce6b99509ab57daa37a6f5b1a1d8fd77f008fa2add4673092f9585ae82d/case.json", "inputs": "artifacts/b2054ce6b99509ab57daa37a6f5b1a1d8fd77f008fa2add4673092f9585ae82d/inputs.npz", "manifest": "artifacts/b2054ce6b99509ab57daa37a6f5b1a1d8fd77f008fa2add4673092f9585ae82d/manifest.json", "model": "artifacts/b2054ce6b99509ab57daa37a6f5b1a1d8fd77f008fa2add4673092f9585ae82d/model.onnx", "outputs": "observations/02827c31df4a37addf4daed75bb32ba2cd624bf9e82ccec7bb72c30a281a61d3/outputs.npz" }, "candidate_group": "d0d87b60b17aeeb5cfc9bd05c417c7a5536eade2727a321fc998d3fe190d1119", "case_id": "b2054ce6b99509ab57daa37a6f5b1a1d8fd77f008fa2add4673092f9585ae82d", "category": "normalization", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 5.551115123125783e-17, "max_budget_ratio": 0.0001187882366455592, "max_rel": 2.2128129406106307e-16, "max_ulp": 1, "mismatched": 0, "name": "scaled_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "l2_unit_or_zero", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 8, 17 ], "backend": "reference", "elements": 136, "expected_dtype": "float64", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 1.1102230246251565e-16, "max_budget_ratio": 0.00023452014248277905, "max_rel": 4.0607715370033657e-16, "max_ulp": 2, "mismatched": 0, "name": "scaled_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 2.220446049250313e-16, "max_budget_ratio": 0.00020185873175002848, "max_rel": 2.220446049250313e-16, "max_ulp": 2, "mismatched": 0, "name": "l2_unit_or_zero", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 8, 17 ], "backend": "ort", "elements": 136, "expected_dtype": "float64", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 1.1102230246251565e-16, "max_budget_ratio": 0.00019989255155491122, "max_rel": 2.437854380935447e-16, "max_ulp": 2, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } } ], "comparison": { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 1.1102230246251565e-16, "max_budget_ratio": 0.00023757647329111835, "max_rel": 4.2558754227539013e-16, "max_ulp": 2, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, "comparison_key": "0118246529f8de5ce30380e3981b76cb234c1d1895fc7e7f3bc1a4878012526d", "contract": "unit_norm", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "float64", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "LpNormalization/tiny/float64/4x17/seed-20260906", "observation_id": "02827c31df4a37addf4daed75bb32ba2cd624bf9e82ccec7bb72c30a281a61d3", "operation": "LpNormalization", "operators": [ "LpNormalization" ], "opset": 19, "optimization": "all", "output_hash": "7e86704e2d7e3eb48574d54aca3d56080b26113167a13d146bc1e3dba5bfd7bf", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.003084, "p95_ms": 0.0032955, "repeats": 7, "samples_ms": [ 0.003167, 0.003333, 0.003084, 0.003208, 0.002875, 0.00275, 0.002541 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.008416, "p95_ms": 0.0094337, "repeats": 7, "samples_ms": [ 0.009584, 0.009083, 0.0085, 0.008334, 0.00825, 0.008416, 0.008209 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-13, "rtol": 1e-12 }, "variant": "tiny", "warnings": [] }, { "artifacts": { "bundle": "observations/264fcaab8046ae0f706ffd3408b5f89348c817718bdd4b48b874c3285611a9f7/reproducer.zip", "bundle_sha256": "c648818102dd4829bbb101c47d67642bf21034cfbd24c12e8caf48ce7605205a", "case": "artifacts/b2054ce6b99509ab57daa37a6f5b1a1d8fd77f008fa2add4673092f9585ae82d/case.json", "inputs": "artifacts/b2054ce6b99509ab57daa37a6f5b1a1d8fd77f008fa2add4673092f9585ae82d/inputs.npz", "manifest": "artifacts/b2054ce6b99509ab57daa37a6f5b1a1d8fd77f008fa2add4673092f9585ae82d/manifest.json", "model": "artifacts/b2054ce6b99509ab57daa37a6f5b1a1d8fd77f008fa2add4673092f9585ae82d/model.onnx", "outputs": "observations/264fcaab8046ae0f706ffd3408b5f89348c817718bdd4b48b874c3285611a9f7/outputs.npz" }, "candidate_group": "d0d87b60b17aeeb5cfc9bd05c417c7a5536eade2727a321fc998d3fe190d1119", "case_id": "b2054ce6b99509ab57daa37a6f5b1a1d8fd77f008fa2add4673092f9585ae82d", "category": "normalization", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 5.551115123125783e-17, "max_budget_ratio": 0.0001187882366455592, "max_rel": 2.2128129406106307e-16, "max_ulp": 1, "mismatched": 0, "name": "scaled_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "l2_unit_or_zero", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 8, 17 ], "backend": "reference", "elements": 136, "expected_dtype": "float64", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 1.1102230246251565e-16, "max_budget_ratio": 0.00023452014248277905, "max_rel": 4.0607715370033657e-16, "max_ulp": 2, "mismatched": 0, "name": "scaled_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 2.220446049250313e-16, "max_budget_ratio": 0.00020185873175002848, "max_rel": 2.220446049250313e-16, "max_ulp": 2, "mismatched": 0, "name": "l2_unit_or_zero", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 8, 17 ], "backend": "ort", "elements": 136, "expected_dtype": "float64", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 1.1102230246251565e-16, "max_budget_ratio": 0.00019989255155491122, "max_rel": 2.437854380935447e-16, "max_ulp": 2, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } } ], "comparison": { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 1.1102230246251565e-16, "max_budget_ratio": 0.00023757647329111835, "max_rel": 4.2558754227539013e-16, "max_ulp": 2, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, "comparison_key": "398b111dee6bec142a2346bafabeb2aa21d099ce340c0e600fe11a3de8427573", "contract": "unit_norm", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "float64", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "LpNormalization/tiny/float64/4x17/seed-20260906", "observation_id": "264fcaab8046ae0f706ffd3408b5f89348c817718bdd4b48b874c3285611a9f7", "operation": "LpNormalization", "operators": [ "LpNormalization" ], "opset": 19, "optimization": "disabled", "output_hash": "7e86704e2d7e3eb48574d54aca3d56080b26113167a13d146bc1e3dba5bfd7bf", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.002834, "p95_ms": 0.0033252, "repeats": 7, "samples_ms": [ 0.003209, 0.003375, 0.003, 0.002834, 0.002708, 0.0025, 0.002584 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.00825, "p95_ms": 0.0091918, "repeats": 7, "samples_ms": [ 0.009292, 0.008958, 0.008375, 0.00825, 0.008042, 0.008166, 0.008041 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-13, "rtol": 1e-12 }, "variant": "tiny", "warnings": [] }, { "artifacts": { "bundle": "observations/c238b065cbadeb6349dc3e6064602f1a756c085405b66ad85f94666ca4637dab/reproducer.zip", "bundle_sha256": "b219ef4b06c5c1b679429b909e3641096da53152a01707ca62c0af8c0e95b44c", "case": "artifacts/159776ca8abb23e9c4f14dd090a7e69ef6cc9d83ce61426f742eaef6bc46e182/case.json", "inputs": "artifacts/159776ca8abb23e9c4f14dd090a7e69ef6cc9d83ce61426f742eaef6bc46e182/inputs.npz", "manifest": "artifacts/159776ca8abb23e9c4f14dd090a7e69ef6cc9d83ce61426f742eaef6bc46e182/manifest.json", "model": "artifacts/159776ca8abb23e9c4f14dd090a7e69ef6cc9d83ce61426f742eaef6bc46e182/model.onnx", "outputs": "observations/c238b065cbadeb6349dc3e6064602f1a756c085405b66ad85f94666ca4637dab/outputs.npz" }, "candidate_group": "334b0b870dd30fff87e7e1cde6d6ac4d197f23da00b3870b3b4250336c2cd539", "case_id": "159776ca8abb23e9c4f14dd090a7e69ef6cc9d83ce61426f742eaef6bc46e182", "category": "normalization", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "actual_dtype": "float16", "actual_shape": [ 4, 257 ], "backend": "reference", "elements": 1028, "expected_dtype": "float16", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0001220703125, "max_budget_ratio": 0.1076889941847943, "max_rel": 0.0009652509652509653, "max_ulp": 1, "mismatched": 0, "name": "scaled_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 0.0003474347718103177, "max_budget_ratio": 0.06316995851096685, "max_rel": 0.0003474347718103177, "max_ulp": 3129414217721, "mismatched": 0, "name": "l2_unit_or_zero", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 257 ], "backend": "reference", "elements": 1028, "expected_dtype": "float16", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 257 ], "backend": "reference", "elements": 1028, "expected_dtype": "float16", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 257 ], "backend": "reference", "elements": 1028, "expected_dtype": "float16", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 8, 257 ], "backend": "reference", "elements": 2056, "expected_dtype": "float16", "expected_shape": [ 8, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 257 ], "backend": "reference", "elements": 1028, "expected_dtype": "float16", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float16", "actual_shape": [ 4, 257 ], "backend": "ort", "elements": 1028, "expected_dtype": "float16", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "scaled_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 2.398384812463128e-05, "max_budget_ratio": 0.004360699659023869, "max_rel": 2.398384812463128e-05, "max_ulp": 123543605412, "mismatched": 0, "name": "l2_unit_or_zero", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 257 ], "backend": "ort", "elements": 1028, "expected_dtype": "float16", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 257 ], "backend": "ort", "elements": 1028, "expected_dtype": "float16", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 257 ], "backend": "ort", "elements": 1028, "expected_dtype": "float16", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 8, 257 ], "backend": "ort", "elements": 2056, "expected_dtype": "float16", "expected_shape": [ 8, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 257 ], "backend": "ort", "elements": 1028, "expected_dtype": "float16", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } } ], "comparison": { "actual_dtype": "float16", "actual_shape": [ 4, 257 ], "elements": 1028, "expected_dtype": "float16", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0001220703125, "max_budget_ratio": 0.10774701002047192, "max_rel": 0.000966183574879227, "max_ulp": 1, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, "comparison_key": "dc200a82476008946df81b2879c0c66e4424cc36f999686c74b6ac80fdabca07", "contract": "unit_norm", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "float16", "shape": [ 4, 257 ] } }, "ir_version": 10, "name": "LpNormalization/wide/float16/4x257/seed-20260906", "observation_id": "c238b065cbadeb6349dc3e6064602f1a756c085405b66ad85f94666ca4637dab", "operation": "LpNormalization", "operators": [ "LpNormalization" ], "opset": 19, "optimization": "all", "output_hash": "f4e9b433aad3dcc96c0a734705f28c7e61b87eaf85c6e3afad2b44c4a8b339f2", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.0045, "p95_ms": 0.005078699999999999, "repeats": 7, "samples_ms": [ 0.00475, 0.004875, 0.0045, 0.005166, 0.004334, 0.004417, 0.004041 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.023708, "p95_ms": 0.0238624, "repeats": 7, "samples_ms": [ 0.023875, 0.023083, 0.023542, 0.023833, 0.023667, 0.023708, 0.023708 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 0.0005, "rtol": 0.005 }, "variant": "wide", "warnings": [ { "backend": "ort", "message": "underflow encountered in cast" }, { "backend": "reference", "message": "underflow encountered in cast" }, { "backend": "reference", "message": "underflow encountered in divide" }, { "backend": "reference", "message": "underflow encountered in multiply" }, { "backend": "reference", "message": "underflow encountered in power" } ] }, { "artifacts": { "bundle": "observations/80bcc93531e17837b01b28fb48700743b108b11ff9dd76d28389edcf051e7c0a/reproducer.zip", "bundle_sha256": "34bd965fd1c27436c9da06613f041ab03df946f1c56f67088649a599f8e0a4bf", "case": "artifacts/159776ca8abb23e9c4f14dd090a7e69ef6cc9d83ce61426f742eaef6bc46e182/case.json", "inputs": "artifacts/159776ca8abb23e9c4f14dd090a7e69ef6cc9d83ce61426f742eaef6bc46e182/inputs.npz", "manifest": "artifacts/159776ca8abb23e9c4f14dd090a7e69ef6cc9d83ce61426f742eaef6bc46e182/manifest.json", "model": "artifacts/159776ca8abb23e9c4f14dd090a7e69ef6cc9d83ce61426f742eaef6bc46e182/model.onnx", "outputs": "observations/80bcc93531e17837b01b28fb48700743b108b11ff9dd76d28389edcf051e7c0a/outputs.npz" }, "candidate_group": "334b0b870dd30fff87e7e1cde6d6ac4d197f23da00b3870b3b4250336c2cd539", "case_id": "159776ca8abb23e9c4f14dd090a7e69ef6cc9d83ce61426f742eaef6bc46e182", "category": "normalization", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "actual_dtype": "float16", "actual_shape": [ 4, 257 ], "backend": "reference", "elements": 1028, "expected_dtype": "float16", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0001220703125, "max_budget_ratio": 0.1076889941847943, "max_rel": 0.0009652509652509653, "max_ulp": 1, "mismatched": 0, "name": "scaled_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 0.0003474347718103177, "max_budget_ratio": 0.06316995851096685, "max_rel": 0.0003474347718103177, "max_ulp": 3129414217721, "mismatched": 0, "name": "l2_unit_or_zero", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 257 ], "backend": "reference", "elements": 1028, "expected_dtype": "float16", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 257 ], "backend": "reference", "elements": 1028, "expected_dtype": "float16", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 257 ], "backend": "reference", "elements": 1028, "expected_dtype": "float16", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 8, 257 ], "backend": "reference", "elements": 2056, "expected_dtype": "float16", "expected_shape": [ 8, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 257 ], "backend": "reference", "elements": 1028, "expected_dtype": "float16", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float16", "actual_shape": [ 4, 257 ], "backend": "ort", "elements": 1028, "expected_dtype": "float16", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "scaled_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 2.398384812463128e-05, "max_budget_ratio": 0.004360699659023869, "max_rel": 2.398384812463128e-05, "max_ulp": 123543605412, "mismatched": 0, "name": "l2_unit_or_zero", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 257 ], "backend": "ort", "elements": 1028, "expected_dtype": "float16", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 257 ], "backend": "ort", "elements": 1028, "expected_dtype": "float16", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 257 ], "backend": "ort", "elements": 1028, "expected_dtype": "float16", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 8, 257 ], "backend": "ort", "elements": 2056, "expected_dtype": "float16", "expected_shape": [ 8, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 257 ], "backend": "ort", "elements": 1028, "expected_dtype": "float16", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } } ], "comparison": { "actual_dtype": "float16", "actual_shape": [ 4, 257 ], "elements": 1028, "expected_dtype": "float16", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0001220703125, "max_budget_ratio": 0.10774701002047192, "max_rel": 0.000966183574879227, "max_ulp": 1, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, "comparison_key": "e38e8a765c92caa8a51a9604ad8e6610cf27d396583ef946f2e3c552cda870a3", "contract": "unit_norm", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "float16", "shape": [ 4, 257 ] } }, "ir_version": 10, "name": "LpNormalization/wide/float16/4x257/seed-20260906", "observation_id": "80bcc93531e17837b01b28fb48700743b108b11ff9dd76d28389edcf051e7c0a", "operation": "LpNormalization", "operators": [ "LpNormalization" ], "opset": 19, "optimization": "disabled", "output_hash": "f4e9b433aad3dcc96c0a734705f28c7e61b87eaf85c6e3afad2b44c4a8b339f2", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.004375, "p95_ms": 0.0053625999999999995, "repeats": 7, "samples_ms": [ 0.004708, 0.005042, 0.004375, 0.0055, 0.00425, 0.004334, 0.004375 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.024875, "p95_ms": 0.07585349999999995, "repeats": 7, "samples_ms": [ 0.026, 0.024875, 0.097041, 0.026416, 0.024584, 0.024167, 0.024208 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 0.0005, "rtol": 0.005 }, "variant": "wide", "warnings": [ { "backend": "ort", "message": "underflow encountered in cast" }, { "backend": "reference", "message": "underflow encountered in cast" }, { "backend": "reference", "message": "underflow encountered in divide" }, { "backend": "reference", "message": "underflow encountered in multiply" }, { "backend": "reference", "message": "underflow encountered in power" } ] }, { "artifacts": { "bundle": "observations/203686dd1fc8a69a02306ae75e5e8ad0785ee5d224063d0c9b72f780f041d14d/reproducer.zip", "bundle_sha256": "870578ea98584bbdb6c5546a48b8f4bda1a428fff9e15920b4f45b6209a4d964", "case": "artifacts/d484196022aeba8ca0c5720db994e5230b663183b91fb8aa1789ab3593e64ebf/case.json", "inputs": "artifacts/d484196022aeba8ca0c5720db994e5230b663183b91fb8aa1789ab3593e64ebf/inputs.npz", "manifest": "artifacts/d484196022aeba8ca0c5720db994e5230b663183b91fb8aa1789ab3593e64ebf/manifest.json", "model": "artifacts/d484196022aeba8ca0c5720db994e5230b663183b91fb8aa1789ab3593e64ebf/model.onnx", "outputs": "observations/203686dd1fc8a69a02306ae75e5e8ad0785ee5d224063d0c9b72f780f041d14d/outputs.npz" }, "candidate_group": "a3eecc87a68f5391e3c565906f27612f0806b4d8879ef80bf96609bee9598e04", "case_id": "d484196022aeba8ca0c5720db994e5230b663183b91fb8aa1789ab3593e64ebf", "category": "normalization", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 257 ], "backend": "reference", "elements": 1028, "expected_dtype": "float32", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 1.4901161193847656e-08, "max_budget_ratio": 0.0065575523185786916, "max_rel": 1.1836856396086783e-07, "max_ulp": 1, "mismatched": 0, "name": "scaled_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 1.9365319792186142e-08, "max_budget_ratio": 0.0017604836174714672, "max_rel": 1.9365319792186142e-08, "max_ulp": 174427294, "mismatched": 0, "name": "l2_unit_or_zero", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 257 ], "backend": "reference", "elements": 1028, "expected_dtype": "float32", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 257 ], "backend": "reference", "elements": 1028, "expected_dtype": "float32", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 257 ], "backend": "reference", "elements": 1028, "expected_dtype": "float32", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 257 ], "backend": "reference", "elements": 2056, "expected_dtype": "float32", "expected_shape": [ 8, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 257 ], "backend": "reference", "elements": 1028, "expected_dtype": "float32", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 257 ], "backend": "ort", "elements": 1028, "expected_dtype": "float32", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 2.9802322387695312e-08, "max_budget_ratio": 0.011803167266625593, "max_rel": 2.1552091080861074e-07, "max_ulp": 2, "mismatched": 0, "name": "scaled_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 1.1624496432105502e-07, "max_budget_ratio": 0.010567724029186818, "max_rel": 1.1624496432105502e-07, "max_ulp": 664984625, "mismatched": 0, "name": "l2_unit_or_zero", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 257 ], "backend": "ort", "elements": 1028, "expected_dtype": "float32", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 257 ], "backend": "ort", "elements": 1028, "expected_dtype": "float32", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 257 ], "backend": "ort", "elements": 1028, "expected_dtype": "float32", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 257 ], "backend": "ort", "elements": 2056, "expected_dtype": "float32", "expected_shape": [ 8, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 257 ], "backend": "ort", "elements": 1028, "expected_dtype": "float32", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 2.9802322387695312e-08, "max_budget_ratio": 0.011803165873478181, "max_rel": 2.1744112672773283e-07, "max_ulp": 2, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } } ], "comparison": { "actual_dtype": "float32", "actual_shape": [ 4, 257 ], "elements": 1028, "expected_dtype": "float32", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 2.9802322387695312e-08, "max_budget_ratio": 0.012259351684781861, "max_rel": 2.174411740083867e-07, "max_ulp": 2, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, "comparison_key": "0d9f9228f9436e815473c24f6751e02a013d6e69169bc2d54dae9df65605f0d2", "contract": "unit_norm", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "float32", "shape": [ 4, 257 ] } }, "ir_version": 10, "name": "LpNormalization/wide/float32/4x257/seed-20260906", "observation_id": "203686dd1fc8a69a02306ae75e5e8ad0785ee5d224063d0c9b72f780f041d14d", "operation": "LpNormalization", "operators": [ "LpNormalization" ], "opset": 19, "optimization": "all", "output_hash": "be12dcd84b49cdd6f822470936376e8a0cef26956d9dd7ce91b4a74353efc306", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.003334, "p95_ms": 0.0036206999999999997, "repeats": 7, "samples_ms": [ 0.003417, 0.003708, 0.003375, 0.003334, 0.003208, 0.00325, 0.003167 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.009333, "p95_ms": 0.0103832, "repeats": 7, "samples_ms": [ 0.010583, 0.009917, 0.0095, 0.009333, 0.009042, 0.008958, 0.008833 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-06, "rtol": 1e-05 }, "variant": "wide", "warnings": [] }, { "artifacts": { "bundle": "observations/b7be40d0017a1bbe65076d0962071943c25f218c5b2b0c0769cca2d8df09c412/reproducer.zip", "bundle_sha256": "6cbe87d86bfae0092812a5c56b7dda4cd8ee76e7d414dde1a71d612dc699aa6c", "case": "artifacts/d484196022aeba8ca0c5720db994e5230b663183b91fb8aa1789ab3593e64ebf/case.json", "inputs": "artifacts/d484196022aeba8ca0c5720db994e5230b663183b91fb8aa1789ab3593e64ebf/inputs.npz", "manifest": "artifacts/d484196022aeba8ca0c5720db994e5230b663183b91fb8aa1789ab3593e64ebf/manifest.json", "model": "artifacts/d484196022aeba8ca0c5720db994e5230b663183b91fb8aa1789ab3593e64ebf/model.onnx", "outputs": "observations/b7be40d0017a1bbe65076d0962071943c25f218c5b2b0c0769cca2d8df09c412/outputs.npz" }, "candidate_group": "a3eecc87a68f5391e3c565906f27612f0806b4d8879ef80bf96609bee9598e04", "case_id": "d484196022aeba8ca0c5720db994e5230b663183b91fb8aa1789ab3593e64ebf", "category": "normalization", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 257 ], "backend": "reference", "elements": 1028, "expected_dtype": "float32", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 1.4901161193847656e-08, "max_budget_ratio": 0.0065575523185786916, "max_rel": 1.1836856396086783e-07, "max_ulp": 1, "mismatched": 0, "name": "scaled_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 1.9365319792186142e-08, "max_budget_ratio": 0.0017604836174714672, "max_rel": 1.9365319792186142e-08, "max_ulp": 174427294, "mismatched": 0, "name": "l2_unit_or_zero", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 257 ], "backend": "reference", "elements": 1028, "expected_dtype": "float32", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 257 ], "backend": "reference", "elements": 1028, "expected_dtype": "float32", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 257 ], "backend": "reference", "elements": 1028, "expected_dtype": "float32", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 257 ], "backend": "reference", "elements": 2056, "expected_dtype": "float32", "expected_shape": [ 8, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 257 ], "backend": "reference", "elements": 1028, "expected_dtype": "float32", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 257 ], "backend": "ort", "elements": 1028, "expected_dtype": "float32", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 2.9802322387695312e-08, "max_budget_ratio": 0.011803167266625593, "max_rel": 2.1552091080861074e-07, "max_ulp": 2, "mismatched": 0, "name": "scaled_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 1.1624496432105502e-07, "max_budget_ratio": 0.010567724029186818, "max_rel": 1.1624496432105502e-07, "max_ulp": 664984625, "mismatched": 0, "name": "l2_unit_or_zero", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 257 ], "backend": "ort", "elements": 1028, "expected_dtype": "float32", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 257 ], "backend": "ort", "elements": 1028, "expected_dtype": "float32", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 257 ], "backend": "ort", "elements": 1028, "expected_dtype": "float32", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 257 ], "backend": "ort", "elements": 2056, "expected_dtype": "float32", "expected_shape": [ 8, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 257 ], "backend": "ort", "elements": 1028, "expected_dtype": "float32", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 2.9802322387695312e-08, "max_budget_ratio": 0.011803165873478181, "max_rel": 2.1744112672773283e-07, "max_ulp": 2, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } } ], "comparison": { "actual_dtype": "float32", "actual_shape": [ 4, 257 ], "elements": 1028, "expected_dtype": "float32", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 2.9802322387695312e-08, "max_budget_ratio": 0.012259351684781861, "max_rel": 2.174411740083867e-07, "max_ulp": 2, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, "comparison_key": "def0af59d075a5b29168acb325293acf6287f04eb2ec44d6de3bce021ea5092b", "contract": "unit_norm", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "float32", "shape": [ 4, 257 ] } }, "ir_version": 10, "name": "LpNormalization/wide/float32/4x257/seed-20260906", "observation_id": "b7be40d0017a1bbe65076d0962071943c25f218c5b2b0c0769cca2d8df09c412", "operation": "LpNormalization", "operators": [ "LpNormalization" ], "opset": 19, "optimization": "disabled", "output_hash": "be12dcd84b49cdd6f822470936376e8a0cef26956d9dd7ce91b4a74353efc306", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.003583, "p95_ms": 0.0039831, "repeats": 7, "samples_ms": [ 0.00375, 0.004083, 0.003667, 0.003583, 0.003417, 0.00325, 0.002958 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.009541, "p95_ms": 0.0110876, "repeats": 7, "samples_ms": [ 0.011375, 0.010417, 0.010083, 0.009541, 0.009125, 0.009083, 0.008834 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-06, "rtol": 1e-05 }, "variant": "wide", "warnings": [] }, { "artifacts": { "bundle": "observations/d01989c1208896dd31165cc1bead69c548fb33cc18a3ee7bde5cd49fcd1f9b81/reproducer.zip", "bundle_sha256": "42d01d3f3d59ebce367560b3cd8fa54893347216e93528ced154314037fc3446", "case": "artifacts/f9758eb62205be091ebdd1f608e4d50ab707ff9df3db8eeb84e5e3dc18a97203/case.json", "inputs": "artifacts/f9758eb62205be091ebdd1f608e4d50ab707ff9df3db8eeb84e5e3dc18a97203/inputs.npz", "manifest": "artifacts/f9758eb62205be091ebdd1f608e4d50ab707ff9df3db8eeb84e5e3dc18a97203/manifest.json", "model": "artifacts/f9758eb62205be091ebdd1f608e4d50ab707ff9df3db8eeb84e5e3dc18a97203/model.onnx", "outputs": "observations/d01989c1208896dd31165cc1bead69c548fb33cc18a3ee7bde5cd49fcd1f9b81/outputs.npz" }, "candidate_group": "bf20072abe0a3e445db8763ac5f5009599c0341385d6af66e5508bdde6bb9e84", "case_id": "f9758eb62205be091ebdd1f608e4d50ab707ff9df3db8eeb84e5e3dc18a97203", "category": "normalization", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4, 257 ], "backend": "reference", "elements": 1028, "expected_dtype": "float64", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 5.551115123125783e-17, "max_budget_ratio": 0.00023855205793475015, "max_rel": 4.2513577442651924e-16, "max_ulp": 3, "mismatched": 0, "name": "scaled_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 1.1102230246251565e-16, "max_budget_ratio": 0.00010092936587501424, "max_rel": 1.1102230246251565e-16, "max_ulp": 1, "mismatched": 0, "name": "l2_unit_or_zero", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 257 ], "backend": "reference", "elements": 1028, "expected_dtype": "float64", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 257 ], "backend": "reference", "elements": 1028, "expected_dtype": "float64", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 257 ], "backend": "reference", "elements": 1028, "expected_dtype": "float64", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 8, 257 ], "backend": "reference", "elements": 2056, "expected_dtype": "float64", "expected_shape": [ 8, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 257 ], "backend": "reference", "elements": 1028, "expected_dtype": "float64", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4, 257 ], "backend": "ort", "elements": 1028, "expected_dtype": "float64", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 5.551115123125783e-17, "max_budget_ratio": 0.0002292305874208993, "max_rel": 4.413446310401558e-16, "max_ulp": 3, "mismatched": 0, "name": "scaled_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 2.220446049250313e-16, "max_budget_ratio": 0.00020185873175002848, "max_rel": 2.220446049250313e-16, "max_ulp": 2, "mismatched": 0, "name": "l2_unit_or_zero", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 257 ], "backend": "ort", "elements": 1028, "expected_dtype": "float64", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 257 ], "backend": "ort", "elements": 1028, "expected_dtype": "float64", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 257 ], "backend": "ort", "elements": 1028, "expected_dtype": "float64", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 8, 257 ], "backend": "ort", "elements": 2056, "expected_dtype": "float64", "expected_shape": [ 8, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 257 ], "backend": "ort", "elements": 1028, "expected_dtype": "float64", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 5.551115123125783e-17, "max_budget_ratio": 0.00021617469346096454, "max_rel": 4.2294523871069936e-16, "max_ulp": 2, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } } ], "comparison": { "actual_dtype": "float64", "actual_shape": [ 4, 257 ], "elements": 1028, "expected_dtype": "float64", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 2.7755575615628914e-17, "max_budget_ratio": 0.00012288654903248685, "max_rel": 2.2521836847853234e-16, "max_ulp": 2, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, "comparison_key": "dbde59e2bbabec4f29c8cf0e79fba5ecab32838ddd9f6374f856dd7fa3cc6110", "contract": "unit_norm", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "float64", "shape": [ 4, 257 ] } }, "ir_version": 10, "name": "LpNormalization/wide/float64/4x257/seed-20260906", "observation_id": "d01989c1208896dd31165cc1bead69c548fb33cc18a3ee7bde5cd49fcd1f9b81", "operation": "LpNormalization", "operators": [ "LpNormalization" ], "opset": 19, "optimization": "all", "output_hash": "ef0b74da092d07494530d4e8414251465b953aa1908ef35eedb345a7ea43c003", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.003625, "p95_ms": 0.0037664, "repeats": 7, "samples_ms": [ 0.003667, 0.003791, 0.003709, 0.0035, 0.00325, 0.003625, 0.003208 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.009375, "p95_ms": 0.0104964, "repeats": 7, "samples_ms": [ 0.010292, 0.010584, 0.009584, 0.009375, 0.009208, 0.009292, 0.009125 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-13, "rtol": 1e-12 }, "variant": "wide", "warnings": [] }, { "artifacts": { "bundle": "observations/8daf0198200408d9b8320452665510f36f939ed3071e9faa365ef04a95ef93b3/reproducer.zip", "bundle_sha256": "45fc3a9e46b0ea09bfb983417125d77ab2bea0255bf495441f2b6f666da5650a", "case": "artifacts/f9758eb62205be091ebdd1f608e4d50ab707ff9df3db8eeb84e5e3dc18a97203/case.json", "inputs": "artifacts/f9758eb62205be091ebdd1f608e4d50ab707ff9df3db8eeb84e5e3dc18a97203/inputs.npz", "manifest": "artifacts/f9758eb62205be091ebdd1f608e4d50ab707ff9df3db8eeb84e5e3dc18a97203/manifest.json", "model": "artifacts/f9758eb62205be091ebdd1f608e4d50ab707ff9df3db8eeb84e5e3dc18a97203/model.onnx", "outputs": "observations/8daf0198200408d9b8320452665510f36f939ed3071e9faa365ef04a95ef93b3/outputs.npz" }, "candidate_group": "bf20072abe0a3e445db8763ac5f5009599c0341385d6af66e5508bdde6bb9e84", "case_id": "f9758eb62205be091ebdd1f608e4d50ab707ff9df3db8eeb84e5e3dc18a97203", "category": "normalization", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4, 257 ], "backend": "reference", "elements": 1028, "expected_dtype": "float64", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 5.551115123125783e-17, "max_budget_ratio": 0.00023855205793475015, "max_rel": 4.2513577442651924e-16, "max_ulp": 3, "mismatched": 0, "name": "scaled_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 1.1102230246251565e-16, "max_budget_ratio": 0.00010092936587501424, "max_rel": 1.1102230246251565e-16, "max_ulp": 1, "mismatched": 0, "name": "l2_unit_or_zero", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 257 ], "backend": "reference", "elements": 1028, "expected_dtype": "float64", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 257 ], "backend": "reference", "elements": 1028, "expected_dtype": "float64", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 257 ], "backend": "reference", "elements": 1028, "expected_dtype": "float64", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 8, 257 ], "backend": "reference", "elements": 2056, "expected_dtype": "float64", "expected_shape": [ 8, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 257 ], "backend": "reference", "elements": 1028, "expected_dtype": "float64", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4, 257 ], "backend": "ort", "elements": 1028, "expected_dtype": "float64", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 5.551115123125783e-17, "max_budget_ratio": 0.0002292305874208993, "max_rel": 4.413446310401558e-16, "max_ulp": 3, "mismatched": 0, "name": "scaled_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 2.220446049250313e-16, "max_budget_ratio": 0.00020185873175002848, "max_rel": 2.220446049250313e-16, "max_ulp": 2, "mismatched": 0, "name": "l2_unit_or_zero", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 257 ], "backend": "ort", "elements": 1028, "expected_dtype": "float64", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 257 ], "backend": "ort", "elements": 1028, "expected_dtype": "float64", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 257 ], "backend": "ort", "elements": 1028, "expected_dtype": "float64", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 8, 257 ], "backend": "ort", "elements": 2056, "expected_dtype": "float64", "expected_shape": [ 8, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 257 ], "backend": "ort", "elements": 1028, "expected_dtype": "float64", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 5.551115123125783e-17, "max_budget_ratio": 0.00021617469346096454, "max_rel": 4.2294523871069936e-16, "max_ulp": 2, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } } ], "comparison": { "actual_dtype": "float64", "actual_shape": [ 4, 257 ], "elements": 1028, "expected_dtype": "float64", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 2.7755575615628914e-17, "max_budget_ratio": 0.00012288654903248685, "max_rel": 2.2521836847853234e-16, "max_ulp": 2, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, "comparison_key": "76d74805d3b83d7b12b214832d21d56bc7e30d3dbbf0b4933950fd0e7465875d", "contract": "unit_norm", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "float64", "shape": [ 4, 257 ] } }, "ir_version": 10, "name": "LpNormalization/wide/float64/4x257/seed-20260906", "observation_id": "8daf0198200408d9b8320452665510f36f939ed3071e9faa365ef04a95ef93b3", "operation": "LpNormalization", "operators": [ "LpNormalization" ], "opset": 19, "optimization": "disabled", "output_hash": "ef0b74da092d07494530d4e8414251465b953aa1908ef35eedb345a7ea43c003", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.003584, "p95_ms": 0.004787499999999999, "repeats": 7, "samples_ms": [ 0.003584, 0.005125, 0.004, 0.003583, 0.003584, 0.0035, 0.003292 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.009125, "p95_ms": 0.010596399999999999, "repeats": 7, "samples_ms": [ 0.010834, 0.010042, 0.009375, 0.009125, 0.009, 0.009041, 0.008916 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-13, "rtol": 1e-12 }, "variant": "wide", "warnings": [] }, { "artifacts": { "bundle": "observations/f0e12c610c95888cb8ffc56e6b5013f01df446256f5138471e73c56d612389e7/reproducer.zip", "bundle_sha256": "f9942281c3f121c0a6b4ecfd036ea209b89f6daed97c406ddc33e1cff300adef", "case": "artifacts/006f56de220e9516af037cd1eabd00777c6634dfff65444ed78037df3e8388b0/case.json", "inputs": "artifacts/006f56de220e9516af037cd1eabd00777c6634dfff65444ed78037df3e8388b0/inputs.npz", "manifest": "artifacts/006f56de220e9516af037cd1eabd00777c6634dfff65444ed78037df3e8388b0/manifest.json", "model": "artifacts/006f56de220e9516af037cd1eabd00777c6634dfff65444ed78037df3e8388b0/model.onnx", "outputs": "observations/f0e12c610c95888cb8ffc56e6b5013f01df446256f5138471e73c56d612389e7/outputs.npz" }, "candidate_group": "060e27dd465253124142f65a844d2dde6824c64ebe282813422c4db7d824dee9", "case_id": "006f56de220e9516af037cd1eabd00777c6634dfff65444ed78037df3e8388b0", "category": "metrics", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "mse_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "backend": "reference", "name": "mse_nonnegative", "passed": true }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 8, 1 ], "backend": "reference", "elements": 8, "expected_dtype": "float16", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "mse_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "backend": "ort", "name": "mse_nonnegative", "passed": true }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 8, 1 ], "backend": "ort", "elements": 8, "expected_dtype": "float16", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } } ], "comparison": { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, "comparison_key": "6f6ff37ae19939c970665d7d89f6a1c5a81c7d5af8e1b187b08b432a2ca3d83e", "contract": "mse", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "target": { "dtype": "float16", "shape": [ 4, 17 ] }, "x": { "dtype": "float16", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "MSE/constant/float16/4x17/seed-20260906", "observation_id": "f0e12c610c95888cb8ffc56e6b5013f01df446256f5138471e73c56d612389e7", "operation": "MSE", "operators": [ "Mul", "ReduceMean", "Sub" ], "opset": 19, "optimization": "all", "output_hash": "a89d75e846628342aea25ededcc4e2cd7222ebb0d5dc0a34d7ba232afa1fabe0", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.004834, "p95_ms": 0.005571400000000001, "repeats": 7, "samples_ms": [ 0.005542, 0.005584, 0.004834, 0.005291, 0.004791, 0.004625, 0.004459 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.022083, "p95_ms": 0.07550799999999995, "repeats": 7, "samples_ms": [ 0.023125, 0.022083, 0.021417, 0.097708, 0.023708, 0.021916, 0.021458 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 0.0005, "rtol": 0.005 }, "variant": "constant", "warnings": [ { "backend": "ort", "message": "underflow encountered in cast" }, { "backend": "reference", "message": "underflow encountered in cast" }, { "backend": "reference", "message": "underflow encountered in divide" } ] }, { "artifacts": { "bundle": "observations/b41483a9b6a279aedb991b3fd9182a9e729685c8d0291eeb40218629779c9ff1/reproducer.zip", "bundle_sha256": "f38c0eb67007225c0409f5bbdb9b3678c8b9bddfec27729e401ca5a11f70aaaa", "case": "artifacts/006f56de220e9516af037cd1eabd00777c6634dfff65444ed78037df3e8388b0/case.json", "inputs": "artifacts/006f56de220e9516af037cd1eabd00777c6634dfff65444ed78037df3e8388b0/inputs.npz", "manifest": "artifacts/006f56de220e9516af037cd1eabd00777c6634dfff65444ed78037df3e8388b0/manifest.json", "model": "artifacts/006f56de220e9516af037cd1eabd00777c6634dfff65444ed78037df3e8388b0/model.onnx", "outputs": "observations/b41483a9b6a279aedb991b3fd9182a9e729685c8d0291eeb40218629779c9ff1/outputs.npz" }, "candidate_group": "060e27dd465253124142f65a844d2dde6824c64ebe282813422c4db7d824dee9", "case_id": "006f56de220e9516af037cd1eabd00777c6634dfff65444ed78037df3e8388b0", "category": "metrics", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "mse_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "backend": "reference", "name": "mse_nonnegative", "passed": true }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 8, 1 ], "backend": "reference", "elements": 8, "expected_dtype": "float16", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "mse_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "backend": "ort", "name": "mse_nonnegative", "passed": true }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 8, 1 ], "backend": "ort", "elements": 8, "expected_dtype": "float16", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } } ], "comparison": { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, "comparison_key": "f7ccc2197b3fbcc4a5ced604946ef31210155e23388031ce85a4add859e02d55", "contract": "mse", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "target": { "dtype": "float16", "shape": [ 4, 17 ] }, "x": { "dtype": "float16", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "MSE/constant/float16/4x17/seed-20260906", "observation_id": "b41483a9b6a279aedb991b3fd9182a9e729685c8d0291eeb40218629779c9ff1", "operation": "MSE", "operators": [ "Mul", "ReduceMean", "Sub" ], "opset": 19, "optimization": "disabled", "output_hash": "a89d75e846628342aea25ededcc4e2cd7222ebb0d5dc0a34d7ba232afa1fabe0", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.004834, "p95_ms": 0.0053536, "repeats": 7, "samples_ms": [ 0.005416, 0.005208, 0.004875, 0.00475, 0.004834, 0.004833, 0.004375 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.020959, "p95_ms": 0.0227089, "repeats": 7, "samples_ms": [ 0.022417, 0.022834, 0.021125, 0.020292, 0.020292, 0.020959, 0.0205 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 0.0005, "rtol": 0.005 }, "variant": "constant", "warnings": [ { "backend": "ort", "message": "underflow encountered in cast" }, { "backend": "reference", "message": "underflow encountered in cast" }, { "backend": "reference", "message": "underflow encountered in divide" } ] }, { "artifacts": { "bundle": "observations/96cbd25622f8b9274f7abd05a899d081dbcc5d689a61fa673f88de9fd271a5f8/reproducer.zip", "bundle_sha256": "32531fb0e24930c70daecb7e9ce6efcc020f2b907b0f342aae48a3388f3242fc", "case": "artifacts/d70c7df04cb37c67c3efedc9c66873b60a1cd8b29c82bc682f97959ee0286762/case.json", "inputs": "artifacts/d70c7df04cb37c67c3efedc9c66873b60a1cd8b29c82bc682f97959ee0286762/inputs.npz", "manifest": "artifacts/d70c7df04cb37c67c3efedc9c66873b60a1cd8b29c82bc682f97959ee0286762/manifest.json", "model": "artifacts/d70c7df04cb37c67c3efedc9c66873b60a1cd8b29c82bc682f97959ee0286762/model.onnx", "outputs": "observations/96cbd25622f8b9274f7abd05a899d081dbcc5d689a61fa673f88de9fd271a5f8/outputs.npz" }, "candidate_group": "e91a0921ab54872789248aeb0018ad2a20adea1386b92d33ac5ed813347a29fe", "case_id": "d70c7df04cb37c67c3efedc9c66873b60a1cd8b29c82bc682f97959ee0286762", "category": "metrics", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 3.637978807091713e-12, "max_budget_ratio": 3.636070048776248e-06, "max_rel": 6.930131316979287e-08, "max_ulp": 1, "mismatched": 0, "name": "mse_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "reference", "name": "mse_nonnegative", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 1 ], "backend": "reference", "elements": 8, "expected_dtype": "float32", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 3.637978807091713e-12, "max_budget_ratio": 3.636070048776248e-06, "max_rel": 6.930131316979287e-08, "max_ulp": 1, "mismatched": 0, "name": "mse_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "mse_nonnegative", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 3.637978807091713e-12, "max_budget_ratio": 3.63597672858941e-06, "max_rel": 6.606936874087705e-08, "max_ulp": 1, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 1.4551915228366852e-11, "max_budget_ratio": 1.4531942794505663e-05, "max_rel": 1.0587973459762206e-07, "max_ulp": 1, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 1 ], "backend": "ort", "elements": 8, "expected_dtype": "float32", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } } ], "comparison": { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, "comparison_key": "58819e3abebc93321e55d9225bf32c5825eb2b0b71fad035b212e0907b1a9f3c", "contract": "mse", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "target": { "dtype": "float32", "shape": [ 4, 17 ] }, "x": { "dtype": "float32", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "MSE/constant/float32/4x17/seed-20260906", "observation_id": "96cbd25622f8b9274f7abd05a899d081dbcc5d689a61fa673f88de9fd271a5f8", "operation": "MSE", "operators": [ "Mul", "ReduceMean", "Sub" ], "opset": 19, "optimization": "all", "output_hash": "06055e813ae1526938f72dfa40e0abab0a51cf6cee576e6b0daa1f70c126f467", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.004084, "p95_ms": 0.0046667, "repeats": 7, "samples_ms": [ 0.004667, 0.004666, 0.004084, 0.004166, 0.003959, 0.004083, 0.003875 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.019, "p95_ms": 0.019975, "repeats": 7, "samples_ms": [ 0.020125, 0.019625, 0.019041, 0.018792, 0.019, 0.018833, 0.018292 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-06, "rtol": 1e-05 }, "variant": "constant", "warnings": [] }, { "artifacts": { "bundle": "observations/9845841c9bdc177d64c4d35610d7d4f3385ff1ca0c07ce458345da999d028490/reproducer.zip", "bundle_sha256": "8e16e76e9f6b2794a3d1685a41cf2bcecd9e5d7e59309f1ba3e59d7060f52f26", "case": "artifacts/d70c7df04cb37c67c3efedc9c66873b60a1cd8b29c82bc682f97959ee0286762/case.json", "inputs": "artifacts/d70c7df04cb37c67c3efedc9c66873b60a1cd8b29c82bc682f97959ee0286762/inputs.npz", "manifest": "artifacts/d70c7df04cb37c67c3efedc9c66873b60a1cd8b29c82bc682f97959ee0286762/manifest.json", "model": "artifacts/d70c7df04cb37c67c3efedc9c66873b60a1cd8b29c82bc682f97959ee0286762/model.onnx", "outputs": "observations/9845841c9bdc177d64c4d35610d7d4f3385ff1ca0c07ce458345da999d028490/outputs.npz" }, "candidate_group": "e91a0921ab54872789248aeb0018ad2a20adea1386b92d33ac5ed813347a29fe", "case_id": "d70c7df04cb37c67c3efedc9c66873b60a1cd8b29c82bc682f97959ee0286762", "category": "metrics", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 3.637978807091713e-12, "max_budget_ratio": 3.636070048776248e-06, "max_rel": 6.930131316979287e-08, "max_ulp": 1, "mismatched": 0, "name": "mse_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "reference", "name": "mse_nonnegative", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 1 ], "backend": "reference", "elements": 8, "expected_dtype": "float32", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 3.637978807091713e-12, "max_budget_ratio": 3.636070048776248e-06, "max_rel": 6.930131316979287e-08, "max_ulp": 1, "mismatched": 0, "name": "mse_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "mse_nonnegative", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 3.637978807091713e-12, "max_budget_ratio": 3.63597672858941e-06, "max_rel": 6.606936874087705e-08, "max_ulp": 1, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 1.4551915228366852e-11, "max_budget_ratio": 1.4531942794505663e-05, "max_rel": 1.0587973459762206e-07, "max_ulp": 1, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 1 ], "backend": "ort", "elements": 8, "expected_dtype": "float32", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } } ], "comparison": { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, "comparison_key": "ecf65901bacba8a7fecafd5d03ce042a1f96867f4b14e5c56c5eefa60fe7e120", "contract": "mse", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "target": { "dtype": "float32", "shape": [ 4, 17 ] }, "x": { "dtype": "float32", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "MSE/constant/float32/4x17/seed-20260906", "observation_id": "9845841c9bdc177d64c4d35610d7d4f3385ff1ca0c07ce458345da999d028490", "operation": "MSE", "operators": [ "Mul", "ReduceMean", "Sub" ], "opset": 19, "optimization": "disabled", "output_hash": "06055e813ae1526938f72dfa40e0abab0a51cf6cee576e6b0daa1f70c126f467", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.004458, "p95_ms": 0.0048294, "repeats": 7, "samples_ms": [ 0.004625, 0.004917, 0.004625, 0.004084, 0.004458, 0.004167, 0.003959 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.019458, "p95_ms": 0.0214832, "repeats": 7, "samples_ms": [ 0.021833, 0.020667, 0.019541, 0.019, 0.018833, 0.019167, 0.019458 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-06, "rtol": 1e-05 }, "variant": "constant", "warnings": [] }, { "artifacts": { "bundle": "observations/12ef5c26f214e368114cb141a2538d995519993024d3de90684116f76b248baa/reproducer.zip", "bundle_sha256": "789765193a42a2394af55000a88cd6ef6d1aa959884135b22e899f985e95b68c", "case": "artifacts/20c9d4f38658e84a092e64262a96fd88dad4e0e16f392cf97c8879b82c80f060/case.json", "inputs": "artifacts/20c9d4f38658e84a092e64262a96fd88dad4e0e16f392cf97c8879b82c80f060/inputs.npz", "manifest": "artifacts/20c9d4f38658e84a092e64262a96fd88dad4e0e16f392cf97c8879b82c80f060/manifest.json", "model": "artifacts/20c9d4f38658e84a092e64262a96fd88dad4e0e16f392cf97c8879b82c80f060/model.onnx", "outputs": "observations/12ef5c26f214e368114cb141a2538d995519993024d3de90684116f76b248baa/outputs.npz" }, "candidate_group": "7c1e8c66ce3a01b59ac646d296617ff72342c63abd0974a8dc0caafdb69fe73f", "case_id": "20c9d4f38658e84a092e64262a96fd88dad4e0e16f392cf97c8879b82c80f060", "category": "metrics", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "mse_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "backend": "reference", "name": "mse_nonnegative", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 8, 1 ], "backend": "reference", "elements": 8, "expected_dtype": "float64", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 2.710505431213761e-20, "max_budget_ratio": 2.7066942237797013e-07, "max_rel": 1.9249829669792719e-16, "max_ulp": 1, "mismatched": 0, "name": "mse_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "backend": "ort", "name": "mse_nonnegative", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 1.3552527156068805e-20, "max_budget_ratio": 1.3541313546383517e-07, "max_rel": 1.6365739910403213e-16, "max_ulp": 1, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 1.3552527156068805e-20, "max_budget_ratio": 1.3541313546383517e-07, "max_rel": 1.6365739910403213e-16, "max_ulp": 1, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 8, 1 ], "backend": "ort", "elements": 8, "expected_dtype": "float64", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } } ], "comparison": { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 2.710505431213761e-20, "max_budget_ratio": 2.7066942237797013e-07, "max_rel": 1.9249829669792719e-16, "max_ulp": 1, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, "comparison_key": "7073539cc869d8e938573bfe7e1a08a4030383a7bd5367a49a8f777c49e10a5e", "contract": "mse", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "target": { "dtype": "float64", "shape": [ 4, 17 ] }, "x": { "dtype": "float64", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "MSE/constant/float64/4x17/seed-20260906", "observation_id": "12ef5c26f214e368114cb141a2538d995519993024d3de90684116f76b248baa", "operation": "MSE", "operators": [ "Mul", "ReduceMean", "Sub" ], "opset": 19, "optimization": "all", "output_hash": "32511ff1fe901aba2fdc861207d44775d279cf9074387eaff7c72260e974f616", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.00425, "p95_ms": 0.0048204, "repeats": 7, "samples_ms": [ 0.004791, 0.004833, 0.004625, 0.00425, 0.004166, 0.003792, 0.003875 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.018, "p95_ms": 0.0193499, "repeats": 7, "samples_ms": [ 0.019625, 0.018708, 0.018, 0.017792, 0.01775, 0.018416, 0.017916 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-13, "rtol": 1e-12 }, "variant": "constant", "warnings": [] }, { "artifacts": { "bundle": "observations/b6215deba6d690d710e9f86ed1329a4cf48d2a7e7e5c606e4b3c67d377dedd13/reproducer.zip", "bundle_sha256": "117f19b289c462424a172339aedde8a7b149f1d9fae35a6845a002d591bff813", "case": "artifacts/20c9d4f38658e84a092e64262a96fd88dad4e0e16f392cf97c8879b82c80f060/case.json", "inputs": "artifacts/20c9d4f38658e84a092e64262a96fd88dad4e0e16f392cf97c8879b82c80f060/inputs.npz", "manifest": "artifacts/20c9d4f38658e84a092e64262a96fd88dad4e0e16f392cf97c8879b82c80f060/manifest.json", "model": "artifacts/20c9d4f38658e84a092e64262a96fd88dad4e0e16f392cf97c8879b82c80f060/model.onnx", "outputs": "observations/b6215deba6d690d710e9f86ed1329a4cf48d2a7e7e5c606e4b3c67d377dedd13/outputs.npz" }, "candidate_group": "7c1e8c66ce3a01b59ac646d296617ff72342c63abd0974a8dc0caafdb69fe73f", "case_id": "20c9d4f38658e84a092e64262a96fd88dad4e0e16f392cf97c8879b82c80f060", "category": "metrics", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "mse_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "backend": "reference", "name": "mse_nonnegative", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 8, 1 ], "backend": "reference", "elements": 8, "expected_dtype": "float64", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 2.710505431213761e-20, "max_budget_ratio": 2.7066942237797013e-07, "max_rel": 1.9249829669792719e-16, "max_ulp": 1, "mismatched": 0, "name": "mse_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "backend": "ort", "name": "mse_nonnegative", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 1.3552527156068805e-20, "max_budget_ratio": 1.3541313546383517e-07, "max_rel": 1.6365739910403213e-16, "max_ulp": 1, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 1.3552527156068805e-20, "max_budget_ratio": 1.3541313546383517e-07, "max_rel": 1.6365739910403213e-16, "max_ulp": 1, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 8, 1 ], "backend": "ort", "elements": 8, "expected_dtype": "float64", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } } ], "comparison": { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 2.710505431213761e-20, "max_budget_ratio": 2.7066942237797013e-07, "max_rel": 1.9249829669792719e-16, "max_ulp": 1, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, "comparison_key": "a591ee8db0a2c40b9207c5b7f18557b76bf3992ba7fae431d5976ef960e0405f", "contract": "mse", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "target": { "dtype": "float64", "shape": [ 4, 17 ] }, "x": { "dtype": "float64", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "MSE/constant/float64/4x17/seed-20260906", "observation_id": "b6215deba6d690d710e9f86ed1329a4cf48d2a7e7e5c606e4b3c67d377dedd13", "operation": "MSE", "operators": [ "Mul", "ReduceMean", "Sub" ], "opset": 19, "optimization": "disabled", "output_hash": "32511ff1fe901aba2fdc861207d44775d279cf9074387eaff7c72260e974f616", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.004166, "p95_ms": 0.0047251, "repeats": 7, "samples_ms": [ 0.004667, 0.00475, 0.004292, 0.004166, 0.004, 0.003959, 0.004042 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.018, "p95_ms": 0.0192625, "repeats": 7, "samples_ms": [ 0.019375, 0.019, 0.018292, 0.018, 0.017667, 0.017917, 0.018 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-13, "rtol": 1e-12 }, "variant": "constant", "warnings": [] }, { "artifacts": { "bundle": "observations/0db362c8ff9703b79c3aff33795798862b198b29e623296675618c199ba8e356/reproducer.zip", "bundle_sha256": "c9ae1c41b847b81627c2d50bc5f16b1a983d32f05f5422b7634407b6d2823e2c", "case": "artifacts/b967f3c172891bb4b563f487962471d6e3ff08f902f686c6fc81636d736d86fa/case.json", "inputs": "artifacts/b967f3c172891bb4b563f487962471d6e3ff08f902f686c6fc81636d736d86fa/inputs.npz", "manifest": "artifacts/b967f3c172891bb4b563f487962471d6e3ff08f902f686c6fc81636d736d86fa/manifest.json", "model": "artifacts/b967f3c172891bb4b563f487962471d6e3ff08f902f686c6fc81636d736d86fa/model.onnx", "outputs": "observations/0db362c8ff9703b79c3aff33795798862b198b29e623296675618c199ba8e356/outputs.npz" }, "candidate_group": "ec0d6e09760b30f7554f7a2d438eb1e69121b06cc609d1f8e59dae7620c9aecc", "case_id": "b967f3c172891bb4b563f487962471d6e3ff08f902f686c6fc81636d736d86fa", "category": "metrics", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "mse_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "backend": "reference", "name": "mse_nonnegative", "passed": true }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 8, 1 ], "backend": "reference", "elements": 8, "expected_dtype": "float16", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "mse_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "backend": "ort", "name": "mse_nonnegative", "passed": true }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 8, 1 ], "backend": "ort", "elements": 8, "expected_dtype": "float16", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } } ], "comparison": { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, "comparison_key": "5b45e649f4f178737b350d193585c12ab1a583378ea1d71f6dad9b88c45e4056", "contract": "mse", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "target": { "dtype": "float16", "shape": [ 4, 17 ] }, "x": { "dtype": "float16", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "MSE/large_offset/float16/4x17/seed-20260906", "observation_id": "0db362c8ff9703b79c3aff33795798862b198b29e623296675618c199ba8e356", "operation": "MSE", "operators": [ "Mul", "ReduceMean", "Sub" ], "opset": 19, "optimization": "all", "output_hash": "416f1d6eaacf00dca96695ed2cb2333e9212ab2b9ecea6c7d42a940a130f7c7c", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.005, "p95_ms": 0.0054044, "repeats": 7, "samples_ms": [ 0.005375, 0.005417, 0.005125, 0.005, 0.004459, 0.004541, 0.0045 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.0195, "p95_ms": 0.024187599999999997, "repeats": 7, "samples_ms": [ 0.021417, 0.025375, 0.019792, 0.019458, 0.019167, 0.0195, 0.018917 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 0.0005, "rtol": 0.005 }, "variant": "large_offset", "warnings": [] }, { "artifacts": { "bundle": "observations/781f222c73c7e0ac4b94e960cfa81e57804b0924a4e7e4131d63c6f57970df2e/reproducer.zip", "bundle_sha256": "6203c0709605dbbe6ba79334dbc6b31038fa697c2e9cd003ab1fbbae421bd8c3", "case": "artifacts/b967f3c172891bb4b563f487962471d6e3ff08f902f686c6fc81636d736d86fa/case.json", "inputs": "artifacts/b967f3c172891bb4b563f487962471d6e3ff08f902f686c6fc81636d736d86fa/inputs.npz", "manifest": "artifacts/b967f3c172891bb4b563f487962471d6e3ff08f902f686c6fc81636d736d86fa/manifest.json", "model": "artifacts/b967f3c172891bb4b563f487962471d6e3ff08f902f686c6fc81636d736d86fa/model.onnx", "outputs": "observations/781f222c73c7e0ac4b94e960cfa81e57804b0924a4e7e4131d63c6f57970df2e/outputs.npz" }, "candidate_group": "ec0d6e09760b30f7554f7a2d438eb1e69121b06cc609d1f8e59dae7620c9aecc", "case_id": "b967f3c172891bb4b563f487962471d6e3ff08f902f686c6fc81636d736d86fa", "category": "metrics", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "mse_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "backend": "reference", "name": "mse_nonnegative", "passed": true }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 8, 1 ], "backend": "reference", "elements": 8, "expected_dtype": "float16", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "mse_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "backend": "ort", "name": "mse_nonnegative", "passed": true }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 8, 1 ], "backend": "ort", "elements": 8, "expected_dtype": "float16", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } } ], "comparison": { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, "comparison_key": "eff5c89bd36b5d0687f1b3c9f8e57c1e028bee39747380e46f2d634c63ff630d", "contract": "mse", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "target": { "dtype": "float16", "shape": [ 4, 17 ] }, "x": { "dtype": "float16", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "MSE/large_offset/float16/4x17/seed-20260906", "observation_id": "781f222c73c7e0ac4b94e960cfa81e57804b0924a4e7e4131d63c6f57970df2e", "operation": "MSE", "operators": [ "Mul", "ReduceMean", "Sub" ], "opset": 19, "optimization": "disabled", "output_hash": "416f1d6eaacf00dca96695ed2cb2333e9212ab2b9ecea6c7d42a940a130f7c7c", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.004708, "p95_ms": 0.0052794, "repeats": 7, "samples_ms": [ 0.00525, 0.005084, 0.004708, 0.005292, 0.004584, 0.004208, 0.004167 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.019917, "p95_ms": 0.024791199999999996, "repeats": 7, "samples_ms": [ 0.021, 0.020458, 0.019917, 0.026416, 0.019709, 0.019291, 0.019 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 0.0005, "rtol": 0.005 }, "variant": "large_offset", "warnings": [] }, { "artifacts": { "bundle": "observations/1ad91fb0411d7182534b61787b5eaf2abc208cc02d181ed22ad7844bdebcb2ab/reproducer.zip", "bundle_sha256": "726a23584f721df5bba097c5261e1a72aeafc883feb5a8c1983b247b080b39d9", "case": "artifacts/0231bd3680675b2c6be71dbc3bea662e650292dab539f80a39b5ccbe628e0d02/case.json", "inputs": "artifacts/0231bd3680675b2c6be71dbc3bea662e650292dab539f80a39b5ccbe628e0d02/inputs.npz", "manifest": "artifacts/0231bd3680675b2c6be71dbc3bea662e650292dab539f80a39b5ccbe628e0d02/manifest.json", "model": "artifacts/0231bd3680675b2c6be71dbc3bea662e650292dab539f80a39b5ccbe628e0d02/model.onnx", "outputs": "observations/1ad91fb0411d7182534b61787b5eaf2abc208cc02d181ed22ad7844bdebcb2ab/outputs.npz" }, "candidate_group": "7faa8f8fddcfc04dd45951dff794af7b51ad4a52f5a2432ba8daa88e9cd107b3", "case_id": "0231bd3680675b2c6be71dbc3bea662e650292dab539f80a39b5ccbe628e0d02", "category": "metrics", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "mse_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "reference", "name": "mse_nonnegative", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 1 ], "backend": "reference", "elements": 8, "expected_dtype": "float32", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "mse_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "mse_nonnegative", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 1 ], "backend": "ort", "elements": 8, "expected_dtype": "float32", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } } ], "comparison": { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, "comparison_key": "dddb47569a8e217571b455ce791218c0687d2794a93a290edd16e4ce40591d71", "contract": "mse", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "target": { "dtype": "float32", "shape": [ 4, 17 ] }, "x": { "dtype": "float32", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "MSE/large_offset/float32/4x17/seed-20260906", "observation_id": "1ad91fb0411d7182534b61787b5eaf2abc208cc02d181ed22ad7844bdebcb2ab", "operation": "MSE", "operators": [ "Mul", "ReduceMean", "Sub" ], "opset": 19, "optimization": "all", "output_hash": "bd7c179a22b1483b3222e3500c9796e5f5d75a3429cc745f847119b176528861", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.00375, "p95_ms": 0.0044082, "repeats": 7, "samples_ms": [ 0.004292, 0.004458, 0.003958, 0.00375, 0.003417, 0.003666, 0.003625 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.018667, "p95_ms": 0.0197252, "repeats": 7, "samples_ms": [ 0.02, 0.019084, 0.018667, 0.018375, 0.018209, 0.018625, 0.018709 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-06, "rtol": 1e-05 }, "variant": "large_offset", "warnings": [] }, { "artifacts": { "bundle": "observations/9b0c6e6eebc375af809a0adb6fb459bdae712ef17f0f6806e266db26b5593675/reproducer.zip", "bundle_sha256": "42efcb7b928f31849828c63628fbd702709700f7ad6a681e18495e99f3a76523", "case": "artifacts/0231bd3680675b2c6be71dbc3bea662e650292dab539f80a39b5ccbe628e0d02/case.json", "inputs": "artifacts/0231bd3680675b2c6be71dbc3bea662e650292dab539f80a39b5ccbe628e0d02/inputs.npz", "manifest": "artifacts/0231bd3680675b2c6be71dbc3bea662e650292dab539f80a39b5ccbe628e0d02/manifest.json", "model": "artifacts/0231bd3680675b2c6be71dbc3bea662e650292dab539f80a39b5ccbe628e0d02/model.onnx", "outputs": "observations/9b0c6e6eebc375af809a0adb6fb459bdae712ef17f0f6806e266db26b5593675/outputs.npz" }, "candidate_group": "7faa8f8fddcfc04dd45951dff794af7b51ad4a52f5a2432ba8daa88e9cd107b3", "case_id": "0231bd3680675b2c6be71dbc3bea662e650292dab539f80a39b5ccbe628e0d02", "category": "metrics", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "mse_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "reference", "name": "mse_nonnegative", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 1 ], "backend": "reference", "elements": 8, "expected_dtype": "float32", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "mse_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "mse_nonnegative", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 1 ], "backend": "ort", "elements": 8, "expected_dtype": "float32", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } } ], "comparison": { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, "comparison_key": "ad67bb29e8ac093e7e187a3f4f3b6ff6acb4d723199f22f57dfcb3bd4c70f6ef", "contract": "mse", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "target": { "dtype": "float32", "shape": [ 4, 17 ] }, "x": { "dtype": "float32", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "MSE/large_offset/float32/4x17/seed-20260906", "observation_id": "9b0c6e6eebc375af809a0adb6fb459bdae712ef17f0f6806e266db26b5593675", "operation": "MSE", "operators": [ "Mul", "ReduceMean", "Sub" ], "opset": 19, "optimization": "disabled", "output_hash": "bd7c179a22b1483b3222e3500c9796e5f5d75a3429cc745f847119b176528861", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.0045, "p95_ms": 0.0050545, "repeats": 7, "samples_ms": [ 0.004792, 0.005167, 0.004625, 0.004333, 0.004166, 0.0045, 0.004125 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.01925, "p95_ms": 0.0229125, "repeats": 7, "samples_ms": [ 0.023625, 0.02125, 0.019792, 0.019208, 0.019, 0.019167, 0.01925 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-06, "rtol": 1e-05 }, "variant": "large_offset", "warnings": [] }, { "artifacts": { "bundle": "observations/ce8e04d8beb06c441c3fb4b2c6720e34c8b0148b9e99ff9c95e9551f220f45fb/reproducer.zip", "bundle_sha256": "cb200ca8de9741c9e78934adbdaeb5c7f352949e82c0287c47fabb7fb0477239", "case": "artifacts/dc2d2de7efdea137669963874c23d30e2ec886d43cd04128cc507d8286eb1b3c/case.json", "inputs": "artifacts/dc2d2de7efdea137669963874c23d30e2ec886d43cd04128cc507d8286eb1b3c/inputs.npz", "manifest": "artifacts/dc2d2de7efdea137669963874c23d30e2ec886d43cd04128cc507d8286eb1b3c/manifest.json", "model": "artifacts/dc2d2de7efdea137669963874c23d30e2ec886d43cd04128cc507d8286eb1b3c/model.onnx", "outputs": "observations/ce8e04d8beb06c441c3fb4b2c6720e34c8b0148b9e99ff9c95e9551f220f45fb/outputs.npz" }, "candidate_group": "0ad1c87c4e5a15930cf14b801a7ce0159992a24a6da612d3163972275736a9e2", "case_id": "dc2d2de7efdea137669963874c23d30e2ec886d43cd04128cc507d8286eb1b3c", "category": "metrics", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "mse_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "backend": "reference", "name": "mse_nonnegative", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 8, 1 ], "backend": "reference", "elements": 8, "expected_dtype": "float64", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "mse_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "backend": "ort", "name": "mse_nonnegative", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 6.776263578034403e-21, "max_budget_ratio": 6.772810107026463e-08, "max_rel": 1.328933885464043e-16, "max_ulp": 1, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 1.3552527156068805e-20, "max_budget_ratio": 1.3541543755092364e-07, "max_rel": 1.670904484591099e-16, "max_ulp": 1, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 8, 1 ], "backend": "ort", "elements": 8, "expected_dtype": "float64", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } } ], "comparison": { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, "comparison_key": "94fbf576fa245c145ada641eb8d9aec047448b5241270940088b504fb81de53e", "contract": "mse", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "target": { "dtype": "float64", "shape": [ 4, 17 ] }, "x": { "dtype": "float64", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "MSE/large_offset/float64/4x17/seed-20260906", "observation_id": "ce8e04d8beb06c441c3fb4b2c6720e34c8b0148b9e99ff9c95e9551f220f45fb", "operation": "MSE", "operators": [ "Mul", "ReduceMean", "Sub" ], "opset": 19, "optimization": "all", "output_hash": "bb08eab977e665c3c7ffe707239e0f0b4f443e0a6c2ed78e5cc8c829f511273d", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.004458, "p95_ms": 0.0045171000000000005, "repeats": 7, "samples_ms": [ 0.004542, 0.004459, 0.004458, 0.004459, 0.004, 0.003792, 0.004042 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.018083, "p95_ms": 0.0193213, "repeats": 7, "samples_ms": [ 0.019459, 0.019, 0.018125, 0.018, 0.018083, 0.01775, 0.017625 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-13, "rtol": 1e-12 }, "variant": "large_offset", "warnings": [] }, { "artifacts": { "bundle": "observations/da57a3c17d096985d6931bf53305c2e620f7d3f699a8877693f47567759e29a5/reproducer.zip", "bundle_sha256": "59ad9f59b648c8f4eb2074ea69bdaade34cd8a21664f98fabe27ad4fc3c45b22", "case": "artifacts/dc2d2de7efdea137669963874c23d30e2ec886d43cd04128cc507d8286eb1b3c/case.json", "inputs": "artifacts/dc2d2de7efdea137669963874c23d30e2ec886d43cd04128cc507d8286eb1b3c/inputs.npz", "manifest": "artifacts/dc2d2de7efdea137669963874c23d30e2ec886d43cd04128cc507d8286eb1b3c/manifest.json", "model": "artifacts/dc2d2de7efdea137669963874c23d30e2ec886d43cd04128cc507d8286eb1b3c/model.onnx", "outputs": "observations/da57a3c17d096985d6931bf53305c2e620f7d3f699a8877693f47567759e29a5/outputs.npz" }, "candidate_group": "0ad1c87c4e5a15930cf14b801a7ce0159992a24a6da612d3163972275736a9e2", "case_id": "dc2d2de7efdea137669963874c23d30e2ec886d43cd04128cc507d8286eb1b3c", "category": "metrics", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "mse_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "backend": "reference", "name": "mse_nonnegative", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 8, 1 ], "backend": "reference", "elements": 8, "expected_dtype": "float64", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "mse_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "backend": "ort", "name": "mse_nonnegative", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 6.776263578034403e-21, "max_budget_ratio": 6.772810107026463e-08, "max_rel": 1.328933885464043e-16, "max_ulp": 1, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 1.3552527156068805e-20, "max_budget_ratio": 1.3541543755092364e-07, "max_rel": 1.670904484591099e-16, "max_ulp": 1, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 8, 1 ], "backend": "ort", "elements": 8, "expected_dtype": "float64", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } } ], "comparison": { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, "comparison_key": "1cded9d11c272b9db63f8be43e9410cb150cef3e8ed8a00b0094a5d7c3d0ca0a", "contract": "mse", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "target": { "dtype": "float64", "shape": [ 4, 17 ] }, "x": { "dtype": "float64", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "MSE/large_offset/float64/4x17/seed-20260906", "observation_id": "da57a3c17d096985d6931bf53305c2e620f7d3f699a8877693f47567759e29a5", "operation": "MSE", "operators": [ "Mul", "ReduceMean", "Sub" ], "opset": 19, "optimization": "disabled", "output_hash": "bb08eab977e665c3c7ffe707239e0f0b4f443e0a6c2ed78e5cc8c829f511273d", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.003666, "p95_ms": 0.0044126999999999994, "repeats": 7, "samples_ms": [ 0.004209, 0.0045, 0.003875, 0.003666, 0.0035, 0.003417, 0.003542 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.018375, "p95_ms": 0.019233, "repeats": 7, "samples_ms": [ 0.019458, 0.018708, 0.0185, 0.018083, 0.017875, 0.018375, 0.01775 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-13, "rtol": 1e-12 }, "variant": "large_offset", "warnings": [] }, { "artifacts": { "bundle": "observations/a3449798dfcf48d753d56c529d8d0147bb1303104c8d0a5cf912d9bd7e44e76d/reproducer.zip", "bundle_sha256": "dc104a656c8e3202c4cc1949b5f5eb82ba6c64b6d8738b7e7fbfecdd5c4640c9", "case": "artifacts/0c3d0a6237ab3fe9068f1569b9ef461005d5849c8a9b16b325c923393ab95fa2/case.json", "inputs": "artifacts/0c3d0a6237ab3fe9068f1569b9ef461005d5849c8a9b16b325c923393ab95fa2/inputs.npz", "manifest": "artifacts/0c3d0a6237ab3fe9068f1569b9ef461005d5849c8a9b16b325c923393ab95fa2/manifest.json", "model": "artifacts/0c3d0a6237ab3fe9068f1569b9ef461005d5849c8a9b16b325c923393ab95fa2/model.onnx", "outputs": "observations/a3449798dfcf48d753d56c529d8d0147bb1303104c8d0a5cf912d9bd7e44e76d/outputs.npz" }, "candidate_group": "78f140d7d26c6c2502d6ca39a655cd455f8dbfdd7e6cd46bc12bb4fa5cd8775c", "case_id": "0c3d0a6237ab3fe9068f1569b9ef461005d5849c8a9b16b325c923393ab95fa2", "category": "metrics", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 1.1920928955078125e-07, "max_budget_ratio": 0.00023800122761033203, "max_rel": 0.0006798096532970768, "max_ulp": 1, "mismatched": 0, "name": "mse_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "backend": "reference", "name": "mse_nonnegative", "passed": true }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 8, 1 ], "backend": "reference", "elements": 8, "expected_dtype": "float16", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "mse_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "backend": "ort", "name": "mse_nonnegative", "passed": true }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 8, 1 ], "backend": "ort", "elements": 8, "expected_dtype": "float16", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } } ], "comparison": { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 1.1920928955078125e-07, "max_budget_ratio": 0.00023800151083359073, "max_rel": 0.0006802721088435374, "max_ulp": 1, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, "comparison_key": "3906c68862d62b1acae662705fb07d9fb6caf6423a108fc38cb7b535e28a130d", "contract": "mse", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "target": { "dtype": "float16", "shape": [ 4, 17 ] }, "x": { "dtype": "float16", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "MSE/ordinary/float16/4x17/seed-20260906", "observation_id": "a3449798dfcf48d753d56c529d8d0147bb1303104c8d0a5cf912d9bd7e44e76d", "operation": "MSE", "operators": [ "Mul", "ReduceMean", "Sub" ], "opset": 19, "optimization": "all", "output_hash": "f69cbd269c37daec2a8208e44670b7dd4d651958e0b061cc9bd35ca2becf893a", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.004917, "p95_ms": 0.0054786, "repeats": 7, "samples_ms": [ 0.005541, 0.005333, 0.005125, 0.004917, 0.004625, 0.004666, 0.004375 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.021375, "p95_ms": 0.022503699999999998, "repeats": 7, "samples_ms": [ 0.022666, 0.021667, 0.022125, 0.021375, 0.020584, 0.020333, 0.020125 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 0.0005, "rtol": 0.005 }, "variant": "ordinary", "warnings": [ { "backend": "reference", "message": "underflow encountered in multiply" } ] }, { "artifacts": { "bundle": "observations/158a9c6744ba5a822379e0ac5c652c5a7df30fbc7cdbc021eacaa6a5379b2316/reproducer.zip", "bundle_sha256": "8c13c1ce0d2d9f6bee54649478f4b5349623a4dc0d2cdbfff791e3629b148cbb", "case": "artifacts/0c3d0a6237ab3fe9068f1569b9ef461005d5849c8a9b16b325c923393ab95fa2/case.json", "inputs": "artifacts/0c3d0a6237ab3fe9068f1569b9ef461005d5849c8a9b16b325c923393ab95fa2/inputs.npz", "manifest": "artifacts/0c3d0a6237ab3fe9068f1569b9ef461005d5849c8a9b16b325c923393ab95fa2/manifest.json", "model": "artifacts/0c3d0a6237ab3fe9068f1569b9ef461005d5849c8a9b16b325c923393ab95fa2/model.onnx", "outputs": "observations/158a9c6744ba5a822379e0ac5c652c5a7df30fbc7cdbc021eacaa6a5379b2316/outputs.npz" }, "candidate_group": "78f140d7d26c6c2502d6ca39a655cd455f8dbfdd7e6cd46bc12bb4fa5cd8775c", "case_id": "0c3d0a6237ab3fe9068f1569b9ef461005d5849c8a9b16b325c923393ab95fa2", "category": "metrics", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 1.1920928955078125e-07, "max_budget_ratio": 0.00023800122761033203, "max_rel": 0.0006798096532970768, "max_ulp": 1, "mismatched": 0, "name": "mse_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "backend": "reference", "name": "mse_nonnegative", "passed": true }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 8, 1 ], "backend": "reference", "elements": 8, "expected_dtype": "float16", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "mse_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "backend": "ort", "name": "mse_nonnegative", "passed": true }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 8, 1 ], "backend": "ort", "elements": 8, "expected_dtype": "float16", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } } ], "comparison": { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 1.1920928955078125e-07, "max_budget_ratio": 0.00023800151083359073, "max_rel": 0.0006802721088435374, "max_ulp": 1, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, "comparison_key": "baa97a44101b1c9df7bb36f6968882a018ded5d2ba22e077c39c2bba0410ee01", "contract": "mse", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "target": { "dtype": "float16", "shape": [ 4, 17 ] }, "x": { "dtype": "float16", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "MSE/ordinary/float16/4x17/seed-20260906", "observation_id": "158a9c6744ba5a822379e0ac5c652c5a7df30fbc7cdbc021eacaa6a5379b2316", "operation": "MSE", "operators": [ "Mul", "ReduceMean", "Sub" ], "opset": 19, "optimization": "disabled", "output_hash": "f69cbd269c37daec2a8208e44670b7dd4d651958e0b061cc9bd35ca2becf893a", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.004958, "p95_ms": 0.0058207, "repeats": 7, "samples_ms": [ 0.005833, 0.005792, 0.005167, 0.004958, 0.0045, 0.00475, 0.004875 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.020334, "p95_ms": 0.0228919, "repeats": 7, "samples_ms": [ 0.023167, 0.02225, 0.021292, 0.020334, 0.020208, 0.019791, 0.019958 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 0.0005, "rtol": 0.005 }, "variant": "ordinary", "warnings": [ { "backend": "reference", "message": "underflow encountered in multiply" } ] }, { "artifacts": { "bundle": "observations/9f54f5a489632541edf5f59251aebfd7b318726e931a5311c086773e0d5a4296/reproducer.zip", "bundle_sha256": "f855065be79b29a6721a9e4763586b9f959af22c99008817e4bb9e040c8dc9dc", "case": "artifacts/38f13d1de755624af76401bab4ead9b3359595836acd514520f0cd5fdde7a9c9/case.json", "inputs": "artifacts/38f13d1de755624af76401bab4ead9b3359595836acd514520f0cd5fdde7a9c9/inputs.npz", "manifest": "artifacts/38f13d1de755624af76401bab4ead9b3359595836acd514520f0cd5fdde7a9c9/manifest.json", "model": "artifacts/38f13d1de755624af76401bab4ead9b3359595836acd514520f0cd5fdde7a9c9/model.onnx", "outputs": "observations/9f54f5a489632541edf5f59251aebfd7b318726e931a5311c086773e0d5a4296/outputs.npz" }, "candidate_group": "35fccc9d1ad896653e772a26410524ee232e71cb7cae3ba978e82278f1ba0112", "case_id": "38f13d1de755624af76401bab4ead9b3359595836acd514520f0cd5fdde7a9c9", "category": "metrics", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 7.275957614183426e-12, "max_budget_ratio": 7.269562804440596e-06, "max_rel": 8.271243862737054e-08, "max_ulp": 1, "mismatched": 0, "name": "mse_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "reference", "name": "mse_nonnegative", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 1 ], "backend": "reference", "elements": 8, "expected_dtype": "float32", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 1.4551915228366852e-11, "max_budget_ratio": 1.4537723982055016e-05, "max_rel": 1.4907198589301982e-07, "max_ulp": 2, "mismatched": 0, "name": "mse_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "mse_nonnegative", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 1.4551915228366852e-11, "max_budget_ratio": 1.453772398416847e-05, "max_rel": 1.4907200811548013e-07, "max_ulp": 2, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 1.4551915228366852e-11, "max_budget_ratio": 1.453772398416847e-05, "max_rel": 1.4907200811548013e-07, "max_ulp": 2, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 1 ], "backend": "ort", "elements": 8, "expected_dtype": "float32", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } } ], "comparison": { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 2.1827872842550278e-11, "max_budget_ratio": 2.180658597149743e-05, "max_rel": 2.2360796217268825e-07, "max_ulp": 3, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, "comparison_key": "b7203baa70e7db421929611764a8d5e5183e6ac3689b848f84c3bd016c53759e", "contract": "mse", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "target": { "dtype": "float32", "shape": [ 4, 17 ] }, "x": { "dtype": "float32", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "MSE/ordinary/float32/4x17/seed-20260906", "observation_id": "9f54f5a489632541edf5f59251aebfd7b318726e931a5311c086773e0d5a4296", "operation": "MSE", "operators": [ "Mul", "ReduceMean", "Sub" ], "opset": 19, "optimization": "all", "output_hash": "6634b5e137bbc01e37df456ba9fce0d8500a15ecd67e997ce956b6e5e0ea8a83", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.0045, "p95_ms": 0.0048627, "repeats": 7, "samples_ms": [ 0.004875, 0.004834, 0.0045, 0.004208, 0.004125, 0.004667, 0.003875 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.018625, "p95_ms": 0.019862499999999998, "repeats": 7, "samples_ms": [ 0.020125, 0.01925, 0.018875, 0.018334, 0.018416, 0.018625, 0.018208 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-06, "rtol": 1e-05 }, "variant": "ordinary", "warnings": [] }, { "artifacts": { "bundle": "observations/ed98bcc55b6dda394c6139df2ce9999524189db9ea6eadaf67b2ebcc1c171033/reproducer.zip", "bundle_sha256": "15d83d3367aa0c44e4bc9101b7b1b33dd4a2ababbfeb69a12041b72d48a5cd20", "case": "artifacts/38f13d1de755624af76401bab4ead9b3359595836acd514520f0cd5fdde7a9c9/case.json", "inputs": "artifacts/38f13d1de755624af76401bab4ead9b3359595836acd514520f0cd5fdde7a9c9/inputs.npz", "manifest": "artifacts/38f13d1de755624af76401bab4ead9b3359595836acd514520f0cd5fdde7a9c9/manifest.json", "model": "artifacts/38f13d1de755624af76401bab4ead9b3359595836acd514520f0cd5fdde7a9c9/model.onnx", "outputs": "observations/ed98bcc55b6dda394c6139df2ce9999524189db9ea6eadaf67b2ebcc1c171033/outputs.npz" }, "candidate_group": "35fccc9d1ad896653e772a26410524ee232e71cb7cae3ba978e82278f1ba0112", "case_id": "38f13d1de755624af76401bab4ead9b3359595836acd514520f0cd5fdde7a9c9", "category": "metrics", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 7.275957614183426e-12, "max_budget_ratio": 7.269562804440596e-06, "max_rel": 8.271243862737054e-08, "max_ulp": 1, "mismatched": 0, "name": "mse_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "reference", "name": "mse_nonnegative", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 1 ], "backend": "reference", "elements": 8, "expected_dtype": "float32", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 1.4551915228366852e-11, "max_budget_ratio": 1.4537723982055016e-05, "max_rel": 1.4907198589301982e-07, "max_ulp": 2, "mismatched": 0, "name": "mse_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "mse_nonnegative", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 1.4551915228366852e-11, "max_budget_ratio": 1.453772398416847e-05, "max_rel": 1.4907200811548013e-07, "max_ulp": 2, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 1.4551915228366852e-11, "max_budget_ratio": 1.453772398416847e-05, "max_rel": 1.4907200811548013e-07, "max_ulp": 2, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 1 ], "backend": "ort", "elements": 8, "expected_dtype": "float32", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } } ], "comparison": { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 2.1827872842550278e-11, "max_budget_ratio": 2.180658597149743e-05, "max_rel": 2.2360796217268825e-07, "max_ulp": 3, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, "comparison_key": "90c27d76c5ab709af3b7162f7a7c88698a1da6463123b82b45a222bb9b38e819", "contract": "mse", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "target": { "dtype": "float32", "shape": [ 4, 17 ] }, "x": { "dtype": "float32", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "MSE/ordinary/float32/4x17/seed-20260906", "observation_id": "ed98bcc55b6dda394c6139df2ce9999524189db9ea6eadaf67b2ebcc1c171033", "operation": "MSE", "operators": [ "Mul", "ReduceMean", "Sub" ], "opset": 19, "optimization": "disabled", "output_hash": "6634b5e137bbc01e37df456ba9fce0d8500a15ecd67e997ce956b6e5e0ea8a83", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.003792, "p95_ms": 0.0043465, "repeats": 7, "samples_ms": [ 0.004084, 0.004459, 0.003791, 0.004042, 0.0035, 0.003792, 0.003667 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.018958, "p95_ms": 0.0200955, "repeats": 7, "samples_ms": [ 0.020208, 0.019833, 0.018958, 0.018667, 0.0185, 0.018959, 0.018792 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-06, "rtol": 1e-05 }, "variant": "ordinary", "warnings": [] }, { "artifacts": { "bundle": "observations/b3bf27cd06c457abbf4e8b4e07b1b19d652385e6c4db2c8f2dc8073e1e5107cc/reproducer.zip", "bundle_sha256": "65f000fab0cb0d15910570b7f538c635ee7aa7b2a6955b4371fd51347dda3d07", "case": "artifacts/4b6844d76e74b37868e68b650019fc62a3b857ae1010ef764abcacfcd6d9dcf7/case.json", "inputs": "artifacts/4b6844d76e74b37868e68b650019fc62a3b857ae1010ef764abcacfcd6d9dcf7/inputs.npz", "manifest": "artifacts/4b6844d76e74b37868e68b650019fc62a3b857ae1010ef764abcacfcd6d9dcf7/manifest.json", "model": "artifacts/4b6844d76e74b37868e68b650019fc62a3b857ae1010ef764abcacfcd6d9dcf7/model.onnx", "outputs": "observations/b3bf27cd06c457abbf4e8b4e07b1b19d652385e6c4db2c8f2dc8073e1e5107cc/outputs.npz" }, "candidate_group": "b0cdee7d67dd08f0921c967e4b804484405d9e2f8896943dd6a6beed66564f8d", "case_id": "4b6844d76e74b37868e68b650019fc62a3b857ae1010ef764abcacfcd6d9dcf7", "category": "metrics", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "mse_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "backend": "reference", "name": "mse_nonnegative", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 8, 1 ], "backend": "reference", "elements": 8, "expected_dtype": "float64", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 1.3552527156068805e-20, "max_budget_ratio": 1.3543648482922014e-07, "max_rel": 2.0673208802985512e-16, "max_ulp": 1, "mismatched": 0, "name": "mse_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "backend": "ort", "name": "mse_nonnegative", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 8, 1 ], "backend": "ort", "elements": 8, "expected_dtype": "float64", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } } ], "comparison": { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 1.3552527156068805e-20, "max_budget_ratio": 1.3543648482922014e-07, "max_rel": 2.0673208802985512e-16, "max_ulp": 1, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, "comparison_key": "a6fd4efcc1eafef714fc4e4667bc510147656bc367e4a92a40f224e5fb0f6f23", "contract": "mse", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "target": { "dtype": "float64", "shape": [ 4, 17 ] }, "x": { "dtype": "float64", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "MSE/ordinary/float64/4x17/seed-20260906", "observation_id": "b3bf27cd06c457abbf4e8b4e07b1b19d652385e6c4db2c8f2dc8073e1e5107cc", "operation": "MSE", "operators": [ "Mul", "ReduceMean", "Sub" ], "opset": 19, "optimization": "all", "output_hash": "54d8255d7f59201fbfd27338467b0bada0675e99211b53fcf65aaea4fdb7bfb0", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.003625, "p95_ms": 0.0041705, "repeats": 7, "samples_ms": [ 0.004083, 0.004208, 0.003709, 0.003625, 0.0035, 0.003417, 0.003542 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.018083, "p95_ms": 0.019291799999999998, "repeats": 7, "samples_ms": [ 0.019542, 0.018708, 0.018083, 0.018, 0.01775, 0.018583, 0.017833 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-13, "rtol": 1e-12 }, "variant": "ordinary", "warnings": [] }, { "artifacts": { "bundle": "observations/e40be607d5b4279d51622b117a1ee0c9bde141f4e3ea458f1270d314caedef50/reproducer.zip", "bundle_sha256": "269e7dc812bacacf757ec71cac9162ac450e66cc76aff0b14a97ce94279723c3", "case": "artifacts/4b6844d76e74b37868e68b650019fc62a3b857ae1010ef764abcacfcd6d9dcf7/case.json", "inputs": "artifacts/4b6844d76e74b37868e68b650019fc62a3b857ae1010ef764abcacfcd6d9dcf7/inputs.npz", "manifest": "artifacts/4b6844d76e74b37868e68b650019fc62a3b857ae1010ef764abcacfcd6d9dcf7/manifest.json", "model": "artifacts/4b6844d76e74b37868e68b650019fc62a3b857ae1010ef764abcacfcd6d9dcf7/model.onnx", "outputs": "observations/e40be607d5b4279d51622b117a1ee0c9bde141f4e3ea458f1270d314caedef50/outputs.npz" }, "candidate_group": "b0cdee7d67dd08f0921c967e4b804484405d9e2f8896943dd6a6beed66564f8d", "case_id": "4b6844d76e74b37868e68b650019fc62a3b857ae1010ef764abcacfcd6d9dcf7", "category": "metrics", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "mse_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "backend": "reference", "name": "mse_nonnegative", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 8, 1 ], "backend": "reference", "elements": 8, "expected_dtype": "float64", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 1.3552527156068805e-20, "max_budget_ratio": 1.3543648482922014e-07, "max_rel": 2.0673208802985512e-16, "max_ulp": 1, "mismatched": 0, "name": "mse_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "backend": "ort", "name": "mse_nonnegative", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 8, 1 ], "backend": "ort", "elements": 8, "expected_dtype": "float64", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } } ], "comparison": { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 1.3552527156068805e-20, "max_budget_ratio": 1.3543648482922014e-07, "max_rel": 2.0673208802985512e-16, "max_ulp": 1, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, "comparison_key": "605bbb46820456c0ead1d846d8a5ccd845adbad753c16107e81c6504a1aa7bab", "contract": "mse", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "target": { "dtype": "float64", "shape": [ 4, 17 ] }, "x": { "dtype": "float64", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "MSE/ordinary/float64/4x17/seed-20260906", "observation_id": "e40be607d5b4279d51622b117a1ee0c9bde141f4e3ea458f1270d314caedef50", "operation": "MSE", "operators": [ "Mul", "ReduceMean", "Sub" ], "opset": 19, "optimization": "disabled", "output_hash": "54d8255d7f59201fbfd27338467b0bada0675e99211b53fcf65aaea4fdb7bfb0", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.004125, "p95_ms": 0.0046624, "repeats": 7, "samples_ms": [ 0.00475, 0.004458, 0.004167, 0.004125, 0.003792, 0.004042, 0.004042 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.018166, "p95_ms": 0.019575, "repeats": 7, "samples_ms": [ 0.019875, 0.018875, 0.018166, 0.018, 0.017625, 0.018334, 0.017792 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-13, "rtol": 1e-12 }, "variant": "ordinary", "warnings": [] }, { "artifacts": { "bundle": "observations/0cda10306ced1b26340c7c5093f80a21630bda5d2c9f2db418697dad2aad0b02/reproducer.zip", "bundle_sha256": "e46f8a90325760256252f3758670775f0761f023adb899dd0e432051c303fe97", "case": "artifacts/645d119f64a054c15055aecbc532e9f9e4266cfa268b6c3c8876d940501d3931/case.json", "inputs": "artifacts/645d119f64a054c15055aecbc532e9f9e4266cfa268b6c3c8876d940501d3931/inputs.npz", "manifest": "artifacts/645d119f64a054c15055aecbc532e9f9e4266cfa268b6c3c8876d940501d3931/manifest.json", "model": "artifacts/645d119f64a054c15055aecbc532e9f9e4266cfa268b6c3c8876d940501d3931/model.onnx", "outputs": "observations/0cda10306ced1b26340c7c5093f80a21630bda5d2c9f2db418697dad2aad0b02/outputs.npz" }, "candidate_group": "3ed6a8ae37e922bdf726cfa8755a8f423e7019e832cb21e01b47a4e72682a1d1", "case_id": "645d119f64a054c15055aecbc532e9f9e4266cfa268b6c3c8876d940501d3931", "category": "metrics", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0625, "max_budget_ratio": 0.012486940845090682, "max_rel": 1.2486943339870247e-07, "max_ulp": 2, "mismatched": 0, "name": "mse_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "reference", "name": "mse_nonnegative", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 1 ], "backend": "reference", "elements": 8, "expected_dtype": "float32", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0625, "max_budget_ratio": 0.009952564485252967, "max_rel": 9.952566070109857e-08, "max_ulp": 1, "mismatched": 0, "name": "mse_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "mse_nonnegative", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.125, "max_budget_ratio": 0.019905130951576925, "max_rel": 1.9905134121291337e-07, "max_ulp": 2, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.125, "max_budget_ratio": 0.019905130951576925, "max_rel": 1.9905134121291337e-07, "max_ulp": 2, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 1 ], "backend": "ort", "elements": 8, "expected_dtype": "float32", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } } ], "comparison": { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.125, "max_budget_ratio": 0.01873041360649169, "max_rel": 1.8730417348661973e-07, "max_ulp": 3, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, "comparison_key": "9c94bee2706f99b766a34b855831d36f6be24f12854b62647a0a4e6d8a6ec724", "contract": "mse", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "target": { "dtype": "float32", "shape": [ 4, 53 ] }, "x": { "dtype": "float32", "shape": [ 4, 53 ] } }, "ir_version": 10, "name": "MSE/random-005/float32/4x53/seed-20260906", "observation_id": "0cda10306ced1b26340c7c5093f80a21630bda5d2c9f2db418697dad2aad0b02", "operation": "MSE", "operators": [ "Mul", "ReduceMean", "Sub" ], "opset": 19, "optimization": "all", "output_hash": "ab7924fd290cef7a27b156483ef9e3bc56f83d5c0014cb961cc429554543621c", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.00475, "p95_ms": 0.0052248, "repeats": 7, "samples_ms": [ 0.00475, 0.005084, 0.00525, 0.005166, 0.004166, 0.00375, 0.003791 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.019417, "p95_ms": 0.0215921, "repeats": 7, "samples_ms": [ 0.021917, 0.020834, 0.020084, 0.019416, 0.01925, 0.019334, 0.019417 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-06, "rtol": 1e-05 }, "variant": "random-005", "warnings": [] }, { "artifacts": { "bundle": "observations/06cd5fdd15339eba00f9c4ed1d147778c1d23e6036d36f25c8d8bddd89097cd7/reproducer.zip", "bundle_sha256": "b70c0b6972cef44e9c54a33a9ea6bf57bd2caccdd0b8d5fae9b9c08f2608b094", "case": "artifacts/645d119f64a054c15055aecbc532e9f9e4266cfa268b6c3c8876d940501d3931/case.json", "inputs": "artifacts/645d119f64a054c15055aecbc532e9f9e4266cfa268b6c3c8876d940501d3931/inputs.npz", "manifest": "artifacts/645d119f64a054c15055aecbc532e9f9e4266cfa268b6c3c8876d940501d3931/manifest.json", "model": "artifacts/645d119f64a054c15055aecbc532e9f9e4266cfa268b6c3c8876d940501d3931/model.onnx", "outputs": "observations/06cd5fdd15339eba00f9c4ed1d147778c1d23e6036d36f25c8d8bddd89097cd7/outputs.npz" }, "candidate_group": "3ed6a8ae37e922bdf726cfa8755a8f423e7019e832cb21e01b47a4e72682a1d1", "case_id": "645d119f64a054c15055aecbc532e9f9e4266cfa268b6c3c8876d940501d3931", "category": "metrics", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0625, "max_budget_ratio": 0.012486940845090682, "max_rel": 1.2486943339870247e-07, "max_ulp": 2, "mismatched": 0, "name": "mse_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "reference", "name": "mse_nonnegative", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 1 ], "backend": "reference", "elements": 8, "expected_dtype": "float32", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0625, "max_budget_ratio": 0.009952564485252967, "max_rel": 9.952566070109857e-08, "max_ulp": 1, "mismatched": 0, "name": "mse_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "mse_nonnegative", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.125, "max_budget_ratio": 0.019905130951576925, "max_rel": 1.9905134121291337e-07, "max_ulp": 2, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.125, "max_budget_ratio": 0.019905130951576925, "max_rel": 1.9905134121291337e-07, "max_ulp": 2, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 1 ], "backend": "ort", "elements": 8, "expected_dtype": "float32", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } } ], "comparison": { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.125, "max_budget_ratio": 0.01873041360649169, "max_rel": 1.8730417348661973e-07, "max_ulp": 3, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, "comparison_key": "44b212683876fcab434c68406aed97fe720c89661e582829f74a22960bc8d948", "contract": "mse", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "target": { "dtype": "float32", "shape": [ 4, 53 ] }, "x": { "dtype": "float32", "shape": [ 4, 53 ] } }, "ir_version": 10, "name": "MSE/random-005/float32/4x53/seed-20260906", "observation_id": "06cd5fdd15339eba00f9c4ed1d147778c1d23e6036d36f25c8d8bddd89097cd7", "operation": "MSE", "operators": [ "Mul", "ReduceMean", "Sub" ], "opset": 19, "optimization": "disabled", "output_hash": "ab7924fd290cef7a27b156483ef9e3bc56f83d5c0014cb961cc429554543621c", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.00425, "p95_ms": 0.0046914, "repeats": 7, "samples_ms": [ 0.004416, 0.004791, 0.004459, 0.00425, 0.004, 0.004041, 0.003875 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.019458, "p95_ms": 0.0215623, "repeats": 7, "samples_ms": [ 0.021416, 0.02075, 0.019458, 0.018958, 0.018917, 0.021625, 0.019125 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-06, "rtol": 1e-05 }, "variant": "random-005", "warnings": [] }, { "artifacts": { "bundle": "observations/622879dbc651b1b45cc2ae18c39a8e8a33ef9e7dfdb553799c587f9604aba9c5/reproducer.zip", "bundle_sha256": "2d3a9953b6e430584a75ccc4841689c14d8948fd46287eff334664c1d4b3b1e9", "case": "artifacts/9a41a63d3682c17d3760e0b49f348c24af2870f6a414e141d2ab14af828ce0b6/case.json", "inputs": "artifacts/9a41a63d3682c17d3760e0b49f348c24af2870f6a414e141d2ab14af828ce0b6/inputs.npz", "manifest": "artifacts/9a41a63d3682c17d3760e0b49f348c24af2870f6a414e141d2ab14af828ce0b6/manifest.json", "model": "artifacts/9a41a63d3682c17d3760e0b49f348c24af2870f6a414e141d2ab14af828ce0b6/model.onnx", "outputs": "observations/622879dbc651b1b45cc2ae18c39a8e8a33ef9e7dfdb553799c587f9604aba9c5/outputs.npz" }, "candidate_group": "ad54b75f1cb8b59a75fbeba6d9af53101f54462530855b9e71facae72b296b70", "case_id": "9a41a63d3682c17d3760e0b49f348c24af2870f6a414e141d2ab14af828ce0b6", "category": "metrics", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.00390625, "max_budget_ratio": 0.00831097570056871, "max_rel": 8.310993383119508e-08, "max_ulp": 1, "mismatched": 0, "name": "mse_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "reference", "name": "mse_nonnegative", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 1 ], "backend": "reference", "elements": 8, "expected_dtype": "float32", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.00390625, "max_budget_ratio": 0.007836264328668097, "max_rel": 7.836280048901522e-08, "max_ulp": 1, "mismatched": 0, "name": "mse_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "mse_nonnegative", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.00390625, "max_budget_ratio": 0.007836263714597759, "max_rel": 7.83627943482872e-08, "max_ulp": 1, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 1 ], "backend": "ort", "elements": 8, "expected_dtype": "float32", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } } ], "comparison": { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.00390625, "max_budget_ratio": 0.008310975009845597, "max_rel": 8.310992692393456e-08, "max_ulp": 1, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, "comparison_key": "8ff4761e882a1de6fe9af875c3c49423db30bb9e8bd9ac6d2c03150835dbe96d", "contract": "mse", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "target": { "dtype": "float32", "shape": [ 4, 125 ] }, "x": { "dtype": "float32", "shape": [ 4, 125 ] } }, "ir_version": 10, "name": "MSE/random-012/float32/4x125/seed-20260906", "observation_id": "622879dbc651b1b45cc2ae18c39a8e8a33ef9e7dfdb553799c587f9604aba9c5", "operation": "MSE", "operators": [ "Mul", "ReduceMean", "Sub" ], "opset": 19, "optimization": "all", "output_hash": "ffe2d34dc95e288d5b9c155b8f27c62abc2aceb3ca8a18799424bbdf0be20a3b", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.004292, "p95_ms": 0.004795499999999999, "repeats": 7, "samples_ms": [ 0.004708, 0.004833, 0.004292, 0.004333, 0.003917, 0.003792, 0.00375 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.019, "p95_ms": 0.0206794, "repeats": 7, "samples_ms": [ 0.020917, 0.020125, 0.019, 0.018916, 0.018584, 0.01925, 0.01875 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-06, "rtol": 1e-05 }, "variant": "random-012", "warnings": [] }, { "artifacts": { "bundle": "observations/40caf7b2157a56d859421470e69103bfcb3ad86e7dea560dec50640378c3679c/reproducer.zip", "bundle_sha256": "65c5b3e4f7902514c30d7d5679624f1419691f7b577cae045557f3cba963dfc8", "case": "artifacts/9a41a63d3682c17d3760e0b49f348c24af2870f6a414e141d2ab14af828ce0b6/case.json", "inputs": "artifacts/9a41a63d3682c17d3760e0b49f348c24af2870f6a414e141d2ab14af828ce0b6/inputs.npz", "manifest": "artifacts/9a41a63d3682c17d3760e0b49f348c24af2870f6a414e141d2ab14af828ce0b6/manifest.json", "model": "artifacts/9a41a63d3682c17d3760e0b49f348c24af2870f6a414e141d2ab14af828ce0b6/model.onnx", "outputs": "observations/40caf7b2157a56d859421470e69103bfcb3ad86e7dea560dec50640378c3679c/outputs.npz" }, "candidate_group": "ad54b75f1cb8b59a75fbeba6d9af53101f54462530855b9e71facae72b296b70", "case_id": "9a41a63d3682c17d3760e0b49f348c24af2870f6a414e141d2ab14af828ce0b6", "category": "metrics", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.00390625, "max_budget_ratio": 0.00831097570056871, "max_rel": 8.310993383119508e-08, "max_ulp": 1, "mismatched": 0, "name": "mse_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "reference", "name": "mse_nonnegative", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 1 ], "backend": "reference", "elements": 8, "expected_dtype": "float32", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.00390625, "max_budget_ratio": 0.007836264328668097, "max_rel": 7.836280048901522e-08, "max_ulp": 1, "mismatched": 0, "name": "mse_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "mse_nonnegative", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.00390625, "max_budget_ratio": 0.007836263714597759, "max_rel": 7.83627943482872e-08, "max_ulp": 1, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 1 ], "backend": "ort", "elements": 8, "expected_dtype": "float32", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } } ], "comparison": { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.00390625, "max_budget_ratio": 0.008310975009845597, "max_rel": 8.310992692393456e-08, "max_ulp": 1, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, "comparison_key": "609bbe2d55c228778ce00eff0b4cbf322d71c8ac2c795df7be816a045bbf9b16", "contract": "mse", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "target": { "dtype": "float32", "shape": [ 4, 125 ] }, "x": { "dtype": "float32", "shape": [ 4, 125 ] } }, "ir_version": 10, "name": "MSE/random-012/float32/4x125/seed-20260906", "observation_id": "40caf7b2157a56d859421470e69103bfcb3ad86e7dea560dec50640378c3679c", "operation": "MSE", "operators": [ "Mul", "ReduceMean", "Sub" ], "opset": 19, "optimization": "disabled", "output_hash": "ffe2d34dc95e288d5b9c155b8f27c62abc2aceb3ca8a18799424bbdf0be20a3b", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.004209, "p95_ms": 0.0047832, "repeats": 7, "samples_ms": [ 0.004833, 0.004667, 0.0045, 0.004208, 0.004209, 0.004125, 0.003958 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.019709, "p95_ms": 0.022525, "repeats": 7, "samples_ms": [ 0.022, 0.021208, 0.02275, 0.019667, 0.019541, 0.019709, 0.0195 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-06, "rtol": 1e-05 }, "variant": "random-012", "warnings": [] }, { "artifacts": { "bundle": "observations/8f10cf91a456d0d76b0036c8c6a134930adfef0e4a28efbcf2d8d01294892374/reproducer.zip", "bundle_sha256": "9a1bc994fb0b008f4c05deb2ce1ac796daa9e87cf8623097bac0df483ccb44d0", "case": "artifacts/ce11d7866f5a48c828c12ef81478be3863df27611146e39cbcbca5c41b258a47/case.json", "inputs": "artifacts/ce11d7866f5a48c828c12ef81478be3863df27611146e39cbcbca5c41b258a47/inputs.npz", "manifest": "artifacts/ce11d7866f5a48c828c12ef81478be3863df27611146e39cbcbca5c41b258a47/manifest.json", "model": "artifacts/ce11d7866f5a48c828c12ef81478be3863df27611146e39cbcbca5c41b258a47/model.onnx", "outputs": "observations/8f10cf91a456d0d76b0036c8c6a134930adfef0e4a28efbcf2d8d01294892374/outputs.npz" }, "candidate_group": "2f0fa00b7f8b6a39d17a6f0f11bbed275d9aa70f6797889b475cd90e83f711eb", "case_id": "ce11d7866f5a48c828c12ef81478be3863df27611146e39cbcbca5c41b258a47", "category": "metrics", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 9.5367431640625e-07, "max_budget_ratio": 0.007699296521436273, "max_rel": 7.761961143312038e-08, "max_ulp": 1, "mismatched": 0, "name": "mse_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "reference", "name": "mse_nonnegative", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 1 ], "backend": "reference", "elements": 8, "expected_dtype": "float32", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 1.9073486328125e-06, "max_budget_ratio": 0.015398593042872546, "max_rel": 1.5523922286624076e-07, "max_ulp": 2, "mismatched": 0, "name": "mse_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "mse_nonnegative", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 9.5367431640625e-07, "max_budget_ratio": 0.01155515571635402, "max_rel": 1.169688048876117e-07, "max_ulp": 1, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 9.5367431640625e-07, "max_budget_ratio": 0.01155515571635402, "max_rel": 1.169688048876117e-07, "max_ulp": 1, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 1 ], "backend": "ort", "elements": 8, "expected_dtype": "float32", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } } ], "comparison": { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 9.5367431640625e-07, "max_budget_ratio": 0.011555157051570411, "max_rel": 1.1696881856931461e-07, "max_ulp": 1, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, "comparison_key": "6f97644250005bd97ac58ef8a5a4c6f2ecfa353b2527470d4042d5329f5577be", "contract": "mse", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "target": { "dtype": "float32", "shape": [ 4, 78 ] }, "x": { "dtype": "float32", "shape": [ 4, 78 ] } }, "ir_version": 10, "name": "MSE/random-019/float32/4x78/seed-20260906", "observation_id": "8f10cf91a456d0d76b0036c8c6a134930adfef0e4a28efbcf2d8d01294892374", "operation": "MSE", "operators": [ "Mul", "ReduceMean", "Sub" ], "opset": 19, "optimization": "all", "output_hash": "966afb910c85dcafae24be609746657133750a135e80904bfad8c1fd2df8979f", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.004458, "p95_ms": 0.0050294, "repeats": 7, "samples_ms": [ 0.005, 0.005042, 0.004959, 0.004458, 0.00425, 0.004292, 0.004292 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.019125, "p95_ms": 0.020829300000000002, "repeats": 7, "samples_ms": [ 0.021042, 0.020333, 0.019583, 0.019125, 0.019083, 0.018667, 0.018875 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-06, "rtol": 1e-05 }, "variant": "random-019", "warnings": [] }, { "artifacts": { "bundle": "observations/a876214571c80b31277cdde7dec2c6034fc063c8c58ea89e32592dddeb8df7e1/reproducer.zip", "bundle_sha256": "2b84c46cac2eb6af05bef4f333dbb1aadbbed4fcba0cfabd7d798cd39759a277", "case": "artifacts/ce11d7866f5a48c828c12ef81478be3863df27611146e39cbcbca5c41b258a47/case.json", "inputs": "artifacts/ce11d7866f5a48c828c12ef81478be3863df27611146e39cbcbca5c41b258a47/inputs.npz", "manifest": "artifacts/ce11d7866f5a48c828c12ef81478be3863df27611146e39cbcbca5c41b258a47/manifest.json", "model": "artifacts/ce11d7866f5a48c828c12ef81478be3863df27611146e39cbcbca5c41b258a47/model.onnx", "outputs": "observations/a876214571c80b31277cdde7dec2c6034fc063c8c58ea89e32592dddeb8df7e1/outputs.npz" }, "candidate_group": "2f0fa00b7f8b6a39d17a6f0f11bbed275d9aa70f6797889b475cd90e83f711eb", "case_id": "ce11d7866f5a48c828c12ef81478be3863df27611146e39cbcbca5c41b258a47", "category": "metrics", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 9.5367431640625e-07, "max_budget_ratio": 0.007699296521436273, "max_rel": 7.761961143312038e-08, "max_ulp": 1, "mismatched": 0, "name": "mse_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "reference", "name": "mse_nonnegative", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 1 ], "backend": "reference", "elements": 8, "expected_dtype": "float32", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 1.9073486328125e-06, "max_budget_ratio": 0.015398593042872546, "max_rel": 1.5523922286624076e-07, "max_ulp": 2, "mismatched": 0, "name": "mse_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "mse_nonnegative", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 9.5367431640625e-07, "max_budget_ratio": 0.01155515571635402, "max_rel": 1.169688048876117e-07, "max_ulp": 1, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 9.5367431640625e-07, "max_budget_ratio": 0.01155515571635402, "max_rel": 1.169688048876117e-07, "max_ulp": 1, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 1 ], "backend": "ort", "elements": 8, "expected_dtype": "float32", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } } ], "comparison": { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 9.5367431640625e-07, "max_budget_ratio": 0.011555157051570411, "max_rel": 1.1696881856931461e-07, "max_ulp": 1, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, "comparison_key": "60b5d599b8f1680bece4efc7da317a94c60b74e8b7f2bdefe71ab1ffb0f1c7fd", "contract": "mse", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "target": { "dtype": "float32", "shape": [ 4, 78 ] }, "x": { "dtype": "float32", "shape": [ 4, 78 ] } }, "ir_version": 10, "name": "MSE/random-019/float32/4x78/seed-20260906", "observation_id": "a876214571c80b31277cdde7dec2c6034fc063c8c58ea89e32592dddeb8df7e1", "operation": "MSE", "operators": [ "Mul", "ReduceMean", "Sub" ], "opset": 19, "optimization": "disabled", "output_hash": "966afb910c85dcafae24be609746657133750a135e80904bfad8c1fd2df8979f", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.004583, "p95_ms": 0.0051252, "repeats": 7, "samples_ms": [ 0.00525, 0.004834, 0.00475, 0.004416, 0.004125, 0.004583, 0.004125 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.018583, "p95_ms": 0.020067, "repeats": 7, "samples_ms": [ 0.020292, 0.019542, 0.019083, 0.018583, 0.018459, 0.018541, 0.018417 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-06, "rtol": 1e-05 }, "variant": "random-019", "warnings": [] }, { "artifacts": { "bundle": "observations/750df791fa0a68f2a10fc18a342ee8078fcb82842bd33dec9be4cf66ecb5a281/reproducer.zip", "bundle_sha256": "0166e7db813961cd4ad6f72be8a724283736c3586dda6e8bef7bfacd4901cd64", "case": "artifacts/deaa12e759568e76df1c9adf8f9377e8fa16ef90d54c7fbf662c2b892213f55c/case.json", "inputs": "artifacts/deaa12e759568e76df1c9adf8f9377e8fa16ef90d54c7fbf662c2b892213f55c/inputs.npz", "manifest": "artifacts/deaa12e759568e76df1c9adf8f9377e8fa16ef90d54c7fbf662c2b892213f55c/manifest.json", "model": "artifacts/deaa12e759568e76df1c9adf8f9377e8fa16ef90d54c7fbf662c2b892213f55c/model.onnx", "outputs": "observations/750df791fa0a68f2a10fc18a342ee8078fcb82842bd33dec9be4cf66ecb5a281/outputs.npz" }, "candidate_group": "c578d8492ea092d4d3d3ffd822efc6329cedb401b1ec39c6d43179e9300f14b8", "case_id": "deaa12e759568e76df1c9adf8f9377e8fa16ef90d54c7fbf662c2b892213f55c", "category": "metrics", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 5.960464477539063e-08, "max_budget_ratio": 0.00011909620746464063, "max_rel": 0.0006277463904582549, "max_ulp": 1, "mismatched": 0, "name": "mse_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "backend": "reference", "name": "mse_nonnegative", "passed": true }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 8, 1 ], "backend": "reference", "elements": 8, "expected_dtype": "float16", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "mse_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "backend": "ort", "name": "mse_nonnegative", "passed": true }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 8, 1 ], "backend": "ort", "elements": 8, "expected_dtype": "float16", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } } ], "comparison": { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 5.960464477539063e-08, "max_budget_ratio": 0.00011909627838421604, "max_rel": 0.000628140703517588, "max_ulp": 1, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, "comparison_key": "0616ebfd6091b302cf2165a478c6fe1580da2648559b4ed0fe0e3f533f295443", "contract": "mse", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "target": { "dtype": "float16", "shape": [ 4, 17 ] }, "x": { "dtype": "float16", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "MSE/tiny/float16/4x17/seed-20260906", "observation_id": "750df791fa0a68f2a10fc18a342ee8078fcb82842bd33dec9be4cf66ecb5a281", "operation": "MSE", "operators": [ "Mul", "ReduceMean", "Sub" ], "opset": 19, "optimization": "all", "output_hash": "7dd76316e8e5ca8c661fec655e3bc4bd7208e85b6a24a6c02b103a0387c479ab", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.005375, "p95_ms": 0.016487699999999994, "repeats": 7, "samples_ms": [ 0.005834, 0.005375, 0.004958, 0.004875, 0.004958, 0.019875, 0.008584 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.021042, "p95_ms": 0.07625799999999995, "repeats": 7, "samples_ms": [ 0.098833, 0.023583, 0.021333, 0.021042, 0.020375, 0.019959, 0.02 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 0.0005, "rtol": 0.005 }, "variant": "tiny", "warnings": [ { "backend": "reference", "message": "underflow encountered in multiply" } ] }, { "artifacts": { "bundle": "observations/56d8d9cbf3e5ea12450111f25b6fd04b153281490aab5ed55331545c2926424f/reproducer.zip", "bundle_sha256": "542c5f0165aeaf8540969712f8555e651949ae3feeb24c323b9e677a534b08c5", "case": "artifacts/deaa12e759568e76df1c9adf8f9377e8fa16ef90d54c7fbf662c2b892213f55c/case.json", "inputs": "artifacts/deaa12e759568e76df1c9adf8f9377e8fa16ef90d54c7fbf662c2b892213f55c/inputs.npz", "manifest": "artifacts/deaa12e759568e76df1c9adf8f9377e8fa16ef90d54c7fbf662c2b892213f55c/manifest.json", "model": "artifacts/deaa12e759568e76df1c9adf8f9377e8fa16ef90d54c7fbf662c2b892213f55c/model.onnx", "outputs": "observations/56d8d9cbf3e5ea12450111f25b6fd04b153281490aab5ed55331545c2926424f/outputs.npz" }, "candidate_group": "c578d8492ea092d4d3d3ffd822efc6329cedb401b1ec39c6d43179e9300f14b8", "case_id": "deaa12e759568e76df1c9adf8f9377e8fa16ef90d54c7fbf662c2b892213f55c", "category": "metrics", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 5.960464477539063e-08, "max_budget_ratio": 0.00011909620746464063, "max_rel": 0.0006277463904582549, "max_ulp": 1, "mismatched": 0, "name": "mse_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "backend": "reference", "name": "mse_nonnegative", "passed": true }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 8, 1 ], "backend": "reference", "elements": 8, "expected_dtype": "float16", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "mse_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "backend": "ort", "name": "mse_nonnegative", "passed": true }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 8, 1 ], "backend": "ort", "elements": 8, "expected_dtype": "float16", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } } ], "comparison": { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 5.960464477539063e-08, "max_budget_ratio": 0.00011909627838421604, "max_rel": 0.000628140703517588, "max_ulp": 1, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, "comparison_key": "79c8b63133ef611e0a87081b9b7be4e1b9e2e227d22dddf79bb417858ab6e4d1", "contract": "mse", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "target": { "dtype": "float16", "shape": [ 4, 17 ] }, "x": { "dtype": "float16", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "MSE/tiny/float16/4x17/seed-20260906", "observation_id": "56d8d9cbf3e5ea12450111f25b6fd04b153281490aab5ed55331545c2926424f", "operation": "MSE", "operators": [ "Mul", "ReduceMean", "Sub" ], "opset": 19, "optimization": "disabled", "output_hash": "7dd76316e8e5ca8c661fec655e3bc4bd7208e85b6a24a6c02b103a0387c479ab", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.004583, "p95_ms": 0.0050875, "repeats": 7, "samples_ms": [ 0.005125, 0.005, 0.004583, 0.004833, 0.004459, 0.004541, 0.004541 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.020667, "p95_ms": 0.022075099999999997, "repeats": 7, "samples_ms": [ 0.02225, 0.021667, 0.021, 0.020667, 0.020417, 0.020083, 0.020042 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 0.0005, "rtol": 0.005 }, "variant": "tiny", "warnings": [ { "backend": "reference", "message": "underflow encountered in multiply" } ] }, { "artifacts": { "bundle": "observations/3d51cebdbfd122d9543b17fb843eb99c193603635826a85a7c1ee7914b1b9d42/reproducer.zip", "bundle_sha256": "f1b3fc3193778e5da11288e47ddde3700588b3cdb6d579b9633f7267b8cb8194", "case": "artifacts/e520b850b55792195dd967de675ddcaf98485fb3d4e54f8938428ccca43f39e8/case.json", "inputs": "artifacts/e520b850b55792195dd967de675ddcaf98485fb3d4e54f8938428ccca43f39e8/inputs.npz", "manifest": "artifacts/e520b850b55792195dd967de675ddcaf98485fb3d4e54f8938428ccca43f39e8/manifest.json", "model": "artifacts/e520b850b55792195dd967de675ddcaf98485fb3d4e54f8938428ccca43f39e8/model.onnx", "outputs": "observations/3d51cebdbfd122d9543b17fb843eb99c193603635826a85a7c1ee7914b1b9d42/outputs.npz" }, "candidate_group": "23bcfedb1b668a5c70b84bc87a484834007d74386c98bec7f8052bcdc49eed26", "case_id": "e520b850b55792195dd967de675ddcaf98485fb3d4e54f8938428ccca43f39e8", "category": "metrics", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 7.275957614183426e-12, "max_budget_ratio": 7.268738874367607e-06, "max_rel": 7.326352979583798e-08, "max_ulp": 1, "mismatched": 0, "name": "mse_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "reference", "name": "mse_nonnegative", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 1 ], "backend": "reference", "elements": 8, "expected_dtype": "float32", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 7.275957614183426e-12, "max_budget_ratio": 7.268738874367607e-06, "max_rel": 7.326352979583798e-08, "max_ulp": 1, "mismatched": 0, "name": "mse_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "mse_nonnegative", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 1.4551915228366852e-11, "max_budget_ratio": 1.4534262871090104e-05, "max_rel": 1.1981479747495106e-07, "max_ulp": 2, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 7.275957614183426e-12, "max_budget_ratio": 7.268738874895952e-06, "max_rel": 7.326353516338317e-08, "max_ulp": 1, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 1 ], "backend": "ort", "elements": 8, "expected_dtype": "float32", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } } ], "comparison": { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 1.4551915228366852e-11, "max_budget_ratio": 1.4534262873202554e-05, "max_rel": 1.1981481183053848e-07, "max_ulp": 2, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, "comparison_key": "45376efdb5943fbe24fcc96030226f6027f50f44a85a8ff2943596c04f872de6", "contract": "mse", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "target": { "dtype": "float32", "shape": [ 4, 17 ] }, "x": { "dtype": "float32", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "MSE/tiny/float32/4x17/seed-20260906", "observation_id": "3d51cebdbfd122d9543b17fb843eb99c193603635826a85a7c1ee7914b1b9d42", "operation": "MSE", "operators": [ "Mul", "ReduceMean", "Sub" ], "opset": 19, "optimization": "all", "output_hash": "7752e2fa7a521350e7f3cfa6be8f1c2c63e660fdbf3b6b344d4d91ab288ba546", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.004125, "p95_ms": 0.004625, "repeats": 7, "samples_ms": [ 0.004625, 0.004625, 0.004125, 0.004, 0.003875, 0.004208, 0.003792 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.018917, "p95_ms": 0.0202337, "repeats": 7, "samples_ms": [ 0.020459, 0.019708, 0.018917, 0.018459, 0.019, 0.018333, 0.018584 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-06, "rtol": 1e-05 }, "variant": "tiny", "warnings": [] }, { "artifacts": { "bundle": "observations/ec695a579913568d29c9753a7be8f9abc2018cd444ace63576cd426545ace5f6/reproducer.zip", "bundle_sha256": "2776ee1c706cec3fd5c5113d21601ae292606515f662ab77ebdfdfb3a6ba0972", "case": "artifacts/e520b850b55792195dd967de675ddcaf98485fb3d4e54f8938428ccca43f39e8/case.json", "inputs": "artifacts/e520b850b55792195dd967de675ddcaf98485fb3d4e54f8938428ccca43f39e8/inputs.npz", "manifest": "artifacts/e520b850b55792195dd967de675ddcaf98485fb3d4e54f8938428ccca43f39e8/manifest.json", "model": "artifacts/e520b850b55792195dd967de675ddcaf98485fb3d4e54f8938428ccca43f39e8/model.onnx", "outputs": "observations/ec695a579913568d29c9753a7be8f9abc2018cd444ace63576cd426545ace5f6/outputs.npz" }, "candidate_group": "23bcfedb1b668a5c70b84bc87a484834007d74386c98bec7f8052bcdc49eed26", "case_id": "e520b850b55792195dd967de675ddcaf98485fb3d4e54f8938428ccca43f39e8", "category": "metrics", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 7.275957614183426e-12, "max_budget_ratio": 7.268738874367607e-06, "max_rel": 7.326352979583798e-08, "max_ulp": 1, "mismatched": 0, "name": "mse_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "reference", "name": "mse_nonnegative", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 1 ], "backend": "reference", "elements": 8, "expected_dtype": "float32", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 7.275957614183426e-12, "max_budget_ratio": 7.268738874367607e-06, "max_rel": 7.326352979583798e-08, "max_ulp": 1, "mismatched": 0, "name": "mse_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "mse_nonnegative", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 1.4551915228366852e-11, "max_budget_ratio": 1.4534262871090104e-05, "max_rel": 1.1981479747495106e-07, "max_ulp": 2, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 7.275957614183426e-12, "max_budget_ratio": 7.268738874895952e-06, "max_rel": 7.326353516338317e-08, "max_ulp": 1, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 1 ], "backend": "ort", "elements": 8, "expected_dtype": "float32", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } } ], "comparison": { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 1.4551915228366852e-11, "max_budget_ratio": 1.4534262873202554e-05, "max_rel": 1.1981481183053848e-07, "max_ulp": 2, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, "comparison_key": "e1de2d2424f5997b37a778fdc46089d16f893d2f37a81b30eb840f6b8fbb5cbf", "contract": "mse", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "target": { "dtype": "float32", "shape": [ 4, 17 ] }, "x": { "dtype": "float32", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "MSE/tiny/float32/4x17/seed-20260906", "observation_id": "ec695a579913568d29c9753a7be8f9abc2018cd444ace63576cd426545ace5f6", "operation": "MSE", "operators": [ "Mul", "ReduceMean", "Sub" ], "opset": 19, "optimization": "disabled", "output_hash": "7752e2fa7a521350e7f3cfa6be8f1c2c63e660fdbf3b6b344d4d91ab288ba546", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.004333, "p95_ms": 0.0046537, "repeats": 7, "samples_ms": [ 0.004666, 0.004625, 0.004292, 0.004333, 0.004292, 0.004541, 0.004083 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.018208, "p95_ms": 0.0196839, "repeats": 7, "samples_ms": [ 0.019959, 0.019042, 0.018458, 0.018208, 0.018167, 0.018125, 0.017958 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-06, "rtol": 1e-05 }, "variant": "tiny", "warnings": [] }, { "artifacts": { "bundle": "observations/f053621b652cdea39421bdf109de9c79b6bd1e5fea80ef58d24db9ebd278dba9/reproducer.zip", "bundle_sha256": "67aa6ee4cf39841cbdf882bbe22bc2228a53ef13ac2f4ebf91e68661c5c586c2", "case": "artifacts/01c5e50f98bbc32242d74a583c9a9d1d6e63347cfb9d91636372b25f2920fc4b/case.json", "inputs": "artifacts/01c5e50f98bbc32242d74a583c9a9d1d6e63347cfb9d91636372b25f2920fc4b/inputs.npz", "manifest": "artifacts/01c5e50f98bbc32242d74a583c9a9d1d6e63347cfb9d91636372b25f2920fc4b/manifest.json", "model": "artifacts/01c5e50f98bbc32242d74a583c9a9d1d6e63347cfb9d91636372b25f2920fc4b/model.onnx", "outputs": "observations/f053621b652cdea39421bdf109de9c79b6bd1e5fea80ef58d24db9ebd278dba9/outputs.npz" }, "candidate_group": "42238ccca90adfda362af59674dd8b54c1c4be4807e532d51ef8a7648c84ba3b", "case_id": "01c5e50f98bbc32242d74a583c9a9d1d6e63347cfb9d91636372b25f2920fc4b", "category": "metrics", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "mse_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "backend": "reference", "name": "mse_nonnegative", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 8, 1 ], "backend": "reference", "elements": 8, "expected_dtype": "float64", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 2.710505431213761e-20, "max_budget_ratio": 2.707300778917376e-07, "max_rel": 2.28984388523602e-16, "max_ulp": 2, "mismatched": 0, "name": "mse_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "backend": "ort", "name": "mse_nonnegative", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 2.710505431213761e-20, "max_budget_ratio": 2.707300778917376e-07, "max_rel": 2.2898438852360197e-16, "max_ulp": 2, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 2.710505431213761e-20, "max_budget_ratio": 2.707300778917376e-07, "max_rel": 2.2898438852360197e-16, "max_ulp": 2, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 8, 1 ], "backend": "ort", "elements": 8, "expected_dtype": "float64", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } } ], "comparison": { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 2.710505431213761e-20, "max_budget_ratio": 2.707300778917376e-07, "max_rel": 2.28984388523602e-16, "max_ulp": 2, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, "comparison_key": "25d79afc7b9eeb74c277135043112961f03573e87f47849119366c8ac60509f2", "contract": "mse", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "target": { "dtype": "float64", "shape": [ 4, 17 ] }, "x": { "dtype": "float64", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "MSE/tiny/float64/4x17/seed-20260906", "observation_id": "f053621b652cdea39421bdf109de9c79b6bd1e5fea80ef58d24db9ebd278dba9", "operation": "MSE", "operators": [ "Mul", "ReduceMean", "Sub" ], "opset": 19, "optimization": "all", "output_hash": "fd63fa86f50cd4354a776e80498dca5b61f279c44dca59a1a4c79c0997329843", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.004583, "p95_ms": 0.0046537, "repeats": 7, "samples_ms": [ 0.004666, 0.004583, 0.004583, 0.00425, 0.004625, 0.003958, 0.003917 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.01825, "p95_ms": 0.019607899999999998, "repeats": 7, "samples_ms": [ 0.019958, 0.018791, 0.018459, 0.018083, 0.01825, 0.01775, 0.017625 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-13, "rtol": 1e-12 }, "variant": "tiny", "warnings": [] }, { "artifacts": { "bundle": "observations/c4b3a1b8cfdb2d55cb9d48e34f4a2abc2f4e23886648fc7ba24907505ba62da0/reproducer.zip", "bundle_sha256": "6d9240e8048ba887e79e523c693ae714a7fcb1b91c39c14afb1a4549f5edb5c2", "case": "artifacts/01c5e50f98bbc32242d74a583c9a9d1d6e63347cfb9d91636372b25f2920fc4b/case.json", "inputs": "artifacts/01c5e50f98bbc32242d74a583c9a9d1d6e63347cfb9d91636372b25f2920fc4b/inputs.npz", "manifest": "artifacts/01c5e50f98bbc32242d74a583c9a9d1d6e63347cfb9d91636372b25f2920fc4b/manifest.json", "model": "artifacts/01c5e50f98bbc32242d74a583c9a9d1d6e63347cfb9d91636372b25f2920fc4b/model.onnx", "outputs": "observations/c4b3a1b8cfdb2d55cb9d48e34f4a2abc2f4e23886648fc7ba24907505ba62da0/outputs.npz" }, "candidate_group": "42238ccca90adfda362af59674dd8b54c1c4be4807e532d51ef8a7648c84ba3b", "case_id": "01c5e50f98bbc32242d74a583c9a9d1d6e63347cfb9d91636372b25f2920fc4b", "category": "metrics", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "mse_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "backend": "reference", "name": "mse_nonnegative", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 8, 1 ], "backend": "reference", "elements": 8, "expected_dtype": "float64", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 2.710505431213761e-20, "max_budget_ratio": 2.707300778917376e-07, "max_rel": 2.28984388523602e-16, "max_ulp": 2, "mismatched": 0, "name": "mse_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "backend": "ort", "name": "mse_nonnegative", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 2.710505431213761e-20, "max_budget_ratio": 2.707300778917376e-07, "max_rel": 2.2898438852360197e-16, "max_ulp": 2, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 2.710505431213761e-20, "max_budget_ratio": 2.707300778917376e-07, "max_rel": 2.2898438852360197e-16, "max_ulp": 2, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 8, 1 ], "backend": "ort", "elements": 8, "expected_dtype": "float64", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } } ], "comparison": { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 2.710505431213761e-20, "max_budget_ratio": 2.707300778917376e-07, "max_rel": 2.28984388523602e-16, "max_ulp": 2, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, "comparison_key": "673b1fda797d805d952e0d80687d6ef28fcac9ba59fa275016d54a42701ff549", "contract": "mse", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "target": { "dtype": "float64", "shape": [ 4, 17 ] }, "x": { "dtype": "float64", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "MSE/tiny/float64/4x17/seed-20260906", "observation_id": "c4b3a1b8cfdb2d55cb9d48e34f4a2abc2f4e23886648fc7ba24907505ba62da0", "operation": "MSE", "operators": [ "Mul", "ReduceMean", "Sub" ], "opset": 19, "optimization": "disabled", "output_hash": "fd63fa86f50cd4354a776e80498dca5b61f279c44dca59a1a4c79c0997329843", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.004333, "p95_ms": 0.004683799999999999, "repeats": 7, "samples_ms": [ 0.004709, 0.004542, 0.00425, 0.004625, 0.004167, 0.004333, 0.004208 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.017791, "p95_ms": 0.0193287, "repeats": 7, "samples_ms": [ 0.019125, 0.019416, 0.018542, 0.017666, 0.017791, 0.017625, 0.017791 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-13, "rtol": 1e-12 }, "variant": "tiny", "warnings": [] }, { "artifacts": { "bundle": "observations/391fcc6e63590f4248502a5de51c01d3ddcdbf4da108cff5db66c3916341b002/reproducer.zip", "bundle_sha256": "8dc662637965f044ee8698a14fc747a35e9e80eeb91a31c216ba649df2479763", "case": "artifacts/87693aad8fe9225de1b59c962ce2a91a3b3031f6e63e7854cc27203d6d0df5cc/case.json", "inputs": "artifacts/87693aad8fe9225de1b59c962ce2a91a3b3031f6e63e7854cc27203d6d0df5cc/inputs.npz", "manifest": "artifacts/87693aad8fe9225de1b59c962ce2a91a3b3031f6e63e7854cc27203d6d0df5cc/manifest.json", "model": "artifacts/87693aad8fe9225de1b59c962ce2a91a3b3031f6e63e7854cc27203d6d0df5cc/model.onnx", "outputs": "observations/391fcc6e63590f4248502a5de51c01d3ddcdbf4da108cff5db66c3916341b002/outputs.npz" }, "candidate_group": "c896e3786dbb8fea7d7bb72be239bca46013a5fa783e41d31279104eec55834f", "case_id": "87693aad8fe9225de1b59c962ce2a91a3b3031f6e63e7854cc27203d6d0df5cc", "category": "metrics", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 5.960464477539063e-08, "max_budget_ratio": 0.00011908684682204245, "max_rel": 0.0005797101449275362, "max_ulp": 1, "mismatched": 0, "name": "mse_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "backend": "reference", "name": "mse_nonnegative", "passed": true }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 8, 1 ], "backend": "reference", "elements": 8, "expected_dtype": "float16", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "mse_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "backend": "ort", "name": "mse_nonnegative", "passed": true }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 8, 1 ], "backend": "ort", "elements": 8, "expected_dtype": "float16", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } } ], "comparison": { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 5.960464477539063e-08, "max_budget_ratio": 0.00011908691773047009, "max_rel": 0.000580046403712297, "max_ulp": 1, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, "comparison_key": "334cb7de13a15e4a2c1b3d8abdf0bbdfc1ecd96d4a584c15737cf0bd3b04a1ac", "contract": "mse", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "target": { "dtype": "float16", "shape": [ 4, 257 ] }, "x": { "dtype": "float16", "shape": [ 4, 257 ] } }, "ir_version": 10, "name": "MSE/wide/float16/4x257/seed-20260906", "observation_id": "391fcc6e63590f4248502a5de51c01d3ddcdbf4da108cff5db66c3916341b002", "operation": "MSE", "operators": [ "Mul", "ReduceMean", "Sub" ], "opset": 19, "optimization": "all", "output_hash": "b2a4871e75ab5cef136ab783fd7bb3a4221bdfebcc8a67e6842971d7fc3fdc0b", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.005458, "p95_ms": 0.0061582, "repeats": 7, "samples_ms": [ 0.006208, 0.006042, 0.00525, 0.005625, 0.005458, 0.005333, 0.005042 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.031333, "p95_ms": 0.0319413, "repeats": 7, "samples_ms": [ 0.032166, 0.031375, 0.031333, 0.031083, 0.031209, 0.031417, 0.031041 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 0.0005, "rtol": 0.005 }, "variant": "wide", "warnings": [ { "backend": "reference", "message": "underflow encountered in multiply" } ] }, { "artifacts": { "bundle": "observations/bfc70730120054eac20d4580510f6c5aa4cacd3321229fc8b120d2a6f85bafd9/reproducer.zip", "bundle_sha256": "4a98d2337db92e216a133ab6f7103dd59a96d418b5d2eee50a60ac2f24e8e6e9", "case": "artifacts/87693aad8fe9225de1b59c962ce2a91a3b3031f6e63e7854cc27203d6d0df5cc/case.json", "inputs": "artifacts/87693aad8fe9225de1b59c962ce2a91a3b3031f6e63e7854cc27203d6d0df5cc/inputs.npz", "manifest": "artifacts/87693aad8fe9225de1b59c962ce2a91a3b3031f6e63e7854cc27203d6d0df5cc/manifest.json", "model": "artifacts/87693aad8fe9225de1b59c962ce2a91a3b3031f6e63e7854cc27203d6d0df5cc/model.onnx", "outputs": "observations/bfc70730120054eac20d4580510f6c5aa4cacd3321229fc8b120d2a6f85bafd9/outputs.npz" }, "candidate_group": "c896e3786dbb8fea7d7bb72be239bca46013a5fa783e41d31279104eec55834f", "case_id": "87693aad8fe9225de1b59c962ce2a91a3b3031f6e63e7854cc27203d6d0df5cc", "category": "metrics", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 5.960464477539063e-08, "max_budget_ratio": 0.00011908684682204245, "max_rel": 0.0005797101449275362, "max_ulp": 1, "mismatched": 0, "name": "mse_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "backend": "reference", "name": "mse_nonnegative", "passed": true }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 8, 1 ], "backend": "reference", "elements": 8, "expected_dtype": "float16", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "mse_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "backend": "ort", "name": "mse_nonnegative", "passed": true }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 8, 1 ], "backend": "ort", "elements": 8, "expected_dtype": "float16", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } } ], "comparison": { "actual_dtype": "float16", "actual_shape": [ 4, 1 ], "elements": 4, "expected_dtype": "float16", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 5.960464477539063e-08, "max_budget_ratio": 0.00011908691773047009, "max_rel": 0.000580046403712297, "max_ulp": 1, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, "comparison_key": "a0cf8bb0221cc5d06f8f56ac403abb0b409127e89068dc8e0d04870cad3d08cf", "contract": "mse", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "target": { "dtype": "float16", "shape": [ 4, 257 ] }, "x": { "dtype": "float16", "shape": [ 4, 257 ] } }, "ir_version": 10, "name": "MSE/wide/float16/4x257/seed-20260906", "observation_id": "bfc70730120054eac20d4580510f6c5aa4cacd3321229fc8b120d2a6f85bafd9", "operation": "MSE", "operators": [ "Mul", "ReduceMean", "Sub" ], "opset": 19, "optimization": "disabled", "output_hash": "b2a4871e75ab5cef136ab783fd7bb3a4221bdfebcc8a67e6842971d7fc3fdc0b", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.0055, "p95_ms": 0.0064873, "repeats": 7, "samples_ms": [ 0.006166, 0.006625, 0.0055, 0.0055, 0.00525, 0.00525, 0.005042 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.031292, "p95_ms": 0.0339919, "repeats": 7, "samples_ms": [ 0.034417, 0.033, 0.031917, 0.031292, 0.031125, 0.030625, 0.030916 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 0.0005, "rtol": 0.005 }, "variant": "wide", "warnings": [ { "backend": "reference", "message": "underflow encountered in multiply" } ] }, { "artifacts": { "bundle": "observations/f51a8206533a983c851e41c2c39fdaf594001ba89658b056844830fef7655267/reproducer.zip", "bundle_sha256": "6c3a06775a566bd14cd4d8f087f2f465bd81aa81f071054975cc70f408e098c6", "case": "artifacts/a17808a13350c3b49736692a1054f4b8381a5ad3d762b1477811aa3af21bd29a/case.json", "inputs": "artifacts/a17808a13350c3b49736692a1054f4b8381a5ad3d762b1477811aa3af21bd29a/inputs.npz", "manifest": "artifacts/a17808a13350c3b49736692a1054f4b8381a5ad3d762b1477811aa3af21bd29a/manifest.json", "model": "artifacts/a17808a13350c3b49736692a1054f4b8381a5ad3d762b1477811aa3af21bd29a/model.onnx", "outputs": "observations/f51a8206533a983c851e41c2c39fdaf594001ba89658b056844830fef7655267/outputs.npz" }, "candidate_group": "01cfe440957f0b2777a08b73dda845426b2cd3514a913ecb8d544ac67d5a4701", "case_id": "a17808a13350c3b49736692a1054f4b8381a5ad3d762b1477811aa3af21bd29a", "category": "metrics", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 7.275957614183426e-12, "max_budget_ratio": 7.269227906486361e-06, "max_rel": 7.859270642393991e-08, "max_ulp": 1, "mismatched": 0, "name": "mse_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "reference", "name": "mse_nonnegative", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 1 ], "backend": "reference", "elements": 8, "expected_dtype": "float32", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 7.275957614183426e-12, "max_budget_ratio": 7.269227906486361e-06, "max_rel": 7.859270642393991e-08, "max_ulp": 1, "mismatched": 0, "name": "mse_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "mse_nonnegative", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 7.275957614183426e-12, "max_budget_ratio": 7.269227905957945e-06, "max_rel": 7.859270024712688e-08, "max_ulp": 1, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 1 ], "backend": "ort", "elements": 8, "expected_dtype": "float32", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } } ], "comparison": { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, "comparison_key": "d415cb878113a65a37a94a08fa065fec787e2eb7f8f9977e3192e050204e0b32", "contract": "mse", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "target": { "dtype": "float32", "shape": [ 4, 257 ] }, "x": { "dtype": "float32", "shape": [ 4, 257 ] } }, "ir_version": 10, "name": "MSE/wide/float32/4x257/seed-20260906", "observation_id": "f51a8206533a983c851e41c2c39fdaf594001ba89658b056844830fef7655267", "operation": "MSE", "operators": [ "Mul", "ReduceMean", "Sub" ], "opset": 19, "optimization": "all", "output_hash": "6a329590276e290cb58917ace53653b8b0eff621a46a3191af770b850e20a83f", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.00425, "p95_ms": 0.0045, "repeats": 7, "samples_ms": [ 0.0045, 0.00425, 0.0045, 0.00425, 0.004458, 0.004, 0.004125 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.020709, "p95_ms": 0.024462299999999996, "repeats": 7, "samples_ms": [ 0.0255, 0.022041, 0.02075, 0.020709, 0.02025, 0.02, 0.019917 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-06, "rtol": 1e-05 }, "variant": "wide", "warnings": [] }, { "artifacts": { "bundle": "observations/57ecb95191c159223e9fcd6438b1e7e0f19725531993e199edfa636e69f784f7/reproducer.zip", "bundle_sha256": "3321b090c08053168a9c7f06098601cecf8faa45a26f75c0e13484912fdbf44c", "case": "artifacts/a17808a13350c3b49736692a1054f4b8381a5ad3d762b1477811aa3af21bd29a/case.json", "inputs": "artifacts/a17808a13350c3b49736692a1054f4b8381a5ad3d762b1477811aa3af21bd29a/inputs.npz", "manifest": "artifacts/a17808a13350c3b49736692a1054f4b8381a5ad3d762b1477811aa3af21bd29a/manifest.json", "model": "artifacts/a17808a13350c3b49736692a1054f4b8381a5ad3d762b1477811aa3af21bd29a/model.onnx", "outputs": "observations/57ecb95191c159223e9fcd6438b1e7e0f19725531993e199edfa636e69f784f7/outputs.npz" }, "candidate_group": "01cfe440957f0b2777a08b73dda845426b2cd3514a913ecb8d544ac67d5a4701", "case_id": "a17808a13350c3b49736692a1054f4b8381a5ad3d762b1477811aa3af21bd29a", "category": "metrics", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 7.275957614183426e-12, "max_budget_ratio": 7.269227906486361e-06, "max_rel": 7.859270642393991e-08, "max_ulp": 1, "mismatched": 0, "name": "mse_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "reference", "name": "mse_nonnegative", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 1 ], "backend": "reference", "elements": 8, "expected_dtype": "float32", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 7.275957614183426e-12, "max_budget_ratio": 7.269227906486361e-06, "max_rel": 7.859270642393991e-08, "max_ulp": 1, "mismatched": 0, "name": "mse_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "mse_nonnegative", "passed": true }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 7.275957614183426e-12, "max_budget_ratio": 7.269227905957945e-06, "max_rel": 7.859270024712688e-08, "max_ulp": 1, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 1 ], "backend": "ort", "elements": 8, "expected_dtype": "float32", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } } ], "comparison": { "actual_dtype": "float32", "actual_shape": [ 4, 1 ], "elements": 4, "expected_dtype": "float32", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, "comparison_key": "eac0028bbefa9e707f9d7690e047929719b2f6484f35445e64606394cd180e0b", "contract": "mse", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "target": { "dtype": "float32", "shape": [ 4, 257 ] }, "x": { "dtype": "float32", "shape": [ 4, 257 ] } }, "ir_version": 10, "name": "MSE/wide/float32/4x257/seed-20260906", "observation_id": "57ecb95191c159223e9fcd6438b1e7e0f19725531993e199edfa636e69f784f7", "operation": "MSE", "operators": [ "Mul", "ReduceMean", "Sub" ], "opset": 19, "optimization": "disabled", "output_hash": "6a329590276e290cb58917ace53653b8b0eff621a46a3191af770b850e20a83f", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.00425, "p95_ms": 0.0046544, "repeats": 7, "samples_ms": [ 0.004667, 0.004625, 0.004334, 0.004208, 0.004125, 0.004208, 0.00425 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.021209, "p95_ms": 0.023996099999999996, "repeats": 7, "samples_ms": [ 0.021375, 0.024834, 0.0195, 0.022041, 0.0195, 0.021209, 0.020708 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-06, "rtol": 1e-05 }, "variant": "wide", "warnings": [] }, { "artifacts": { "bundle": "observations/f1780a9a5be92c6cc71d40c8ddac55b0933edc759097c7decaaf80f232ee4d74/reproducer.zip", "bundle_sha256": "62449ddd7b184112e643af14b219c7c736a7047a9a9e0cabb0b939fbd63bd4b5", "case": "artifacts/15fcae67de4a0669103e0424383c205845f344ed70aa42b7be505839212e0ab0/case.json", "inputs": "artifacts/15fcae67de4a0669103e0424383c205845f344ed70aa42b7be505839212e0ab0/inputs.npz", "manifest": "artifacts/15fcae67de4a0669103e0424383c205845f344ed70aa42b7be505839212e0ab0/manifest.json", "model": "artifacts/15fcae67de4a0669103e0424383c205845f344ed70aa42b7be505839212e0ab0/model.onnx", "outputs": "observations/f1780a9a5be92c6cc71d40c8ddac55b0933edc759097c7decaaf80f232ee4d74/outputs.npz" }, "candidate_group": "7354c7098b36469f50637c58258c3cbd39060d593e6aa1c33f1640dd6beb483d", "case_id": "15fcae67de4a0669103e0424383c205845f344ed70aa42b7be505839212e0ab0", "category": "metrics", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "mse_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "backend": "reference", "name": "mse_nonnegative", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 8, 1 ], "backend": "reference", "elements": 8, "expected_dtype": "float64", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 2.710505431213761e-20, "max_budget_ratio": 2.7078869660937523e-07, "max_rel": 2.803070498294833e-16, "max_ulp": 2, "mismatched": 0, "name": "mse_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "backend": "ort", "name": "mse_nonnegative", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 1.3552527156068805e-20, "max_budget_ratio": 1.3539434830468762e-07, "max_rel": 1.401535249147417e-16, "max_ulp": 1, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 1.3552527156068805e-20, "max_budget_ratio": 1.3539434830468762e-07, "max_rel": 1.401535249147417e-16, "max_ulp": 1, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 8, 1 ], "backend": "ort", "elements": 8, "expected_dtype": "float64", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } } ], "comparison": { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 2.710505431213761e-20, "max_budget_ratio": 2.7078869660937523e-07, "max_rel": 2.803070498294833e-16, "max_ulp": 2, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, "comparison_key": "8c7a66154225f631832792c0a87a5715f5035cb4bac81f08bc7bff448fbfe345", "contract": "mse", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "target": { "dtype": "float64", "shape": [ 4, 257 ] }, "x": { "dtype": "float64", "shape": [ 4, 257 ] } }, "ir_version": 10, "name": "MSE/wide/float64/4x257/seed-20260906", "observation_id": "f1780a9a5be92c6cc71d40c8ddac55b0933edc759097c7decaaf80f232ee4d74", "operation": "MSE", "operators": [ "Mul", "ReduceMean", "Sub" ], "opset": 19, "optimization": "all", "output_hash": "d0d900cfdfb3d950e36d81e3864adfd16e055a15c913610d7c052e7d830e5315", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.004, "p95_ms": 0.0051456, "repeats": 7, "samples_ms": [ 0.005, 0.005208, 0.004792, 0.003958, 0.004, 0.00375, 0.003791 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.01925, "p95_ms": 0.0203706, "repeats": 7, "samples_ms": [ 0.020583, 0.019875, 0.019583, 0.01925, 0.018584, 0.0185, 0.018291 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-13, "rtol": 1e-12 }, "variant": "wide", "warnings": [] }, { "artifacts": { "bundle": "observations/68833db4e02175b949231d6cde60432d7267a7683c5769d38653dfe22de5e617/reproducer.zip", "bundle_sha256": "ce841a3b750b1881991a56fb66d7d64386ba6380e48039d5f8c90d16aaa019a7", "case": "artifacts/15fcae67de4a0669103e0424383c205845f344ed70aa42b7be505839212e0ab0/case.json", "inputs": "artifacts/15fcae67de4a0669103e0424383c205845f344ed70aa42b7be505839212e0ab0/inputs.npz", "manifest": "artifacts/15fcae67de4a0669103e0424383c205845f344ed70aa42b7be505839212e0ab0/manifest.json", "model": "artifacts/15fcae67de4a0669103e0424383c205845f344ed70aa42b7be505839212e0ab0/model.onnx", "outputs": "observations/68833db4e02175b949231d6cde60432d7267a7683c5769d38653dfe22de5e617/outputs.npz" }, "candidate_group": "7354c7098b36469f50637c58258c3cbd39060d593e6aa1c33f1640dd6beb483d", "case_id": "15fcae67de4a0669103e0424383c205845f344ed70aa42b7be505839212e0ab0", "category": "metrics", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "mse_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "backend": "reference", "name": "mse_nonnegative", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 8, 1 ], "backend": "reference", "elements": 8, "expected_dtype": "float64", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 2.710505431213761e-20, "max_budget_ratio": 2.7078869660937523e-07, "max_rel": 2.803070498294833e-16, "max_ulp": 2, "mismatched": 0, "name": "mse_float64_oracle", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "backend": "ort", "name": "mse_nonnegative", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 1.3552527156068805e-20, "max_budget_ratio": 1.3539434830468762e-07, "max_rel": 1.401535249147417e-16, "max_ulp": 1, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 1.3552527156068805e-20, "max_budget_ratio": 1.3539434830468762e-07, "max_rel": 1.401535249147417e-16, "max_ulp": 1, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 8, 1 ], "backend": "ort", "elements": 8, "expected_dtype": "float64", "expected_shape": [ 8, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } } ], "comparison": { "actual_dtype": "float64", "actual_shape": [ 4, 1 ], "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4, 1 ], "first_mismatch": null, "max_abs": 2.710505431213761e-20, "max_budget_ratio": 2.7078869660937523e-07, "max_rel": 2.803070498294833e-16, "max_ulp": 2, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, "comparison_key": "32b3f111defdd68844739c23e0ab0427029499ac8fad64451928add483202d51", "contract": "mse", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "target": { "dtype": "float64", "shape": [ 4, 257 ] }, "x": { "dtype": "float64", "shape": [ 4, 257 ] } }, "ir_version": 10, "name": "MSE/wide/float64/4x257/seed-20260906", "observation_id": "68833db4e02175b949231d6cde60432d7267a7683c5769d38653dfe22de5e617", "operation": "MSE", "operators": [ "Mul", "ReduceMean", "Sub" ], "opset": 19, "optimization": "disabled", "output_hash": "d0d900cfdfb3d950e36d81e3864adfd16e055a15c913610d7c052e7d830e5315", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.005, "p95_ms": 0.005266799999999999, "repeats": 7, "samples_ms": [ 0.005208, 0.005292, 0.005, 0.005083, 0.004708, 0.004417, 0.004292 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.019459, "p95_ms": 0.0204501, "repeats": 7, "samples_ms": [ 0.020625, 0.020042, 0.019459, 0.019208, 0.01925, 0.019583, 0.018666 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-13, "rtol": 1e-12 }, "variant": "wide", "warnings": [] }, { "artifacts": { "bundle": "observations/d61707df367e219f7fc55d4c4333eb4982378e1e357ebdd9de83cea046cb1ee2/reproducer.zip", "bundle_sha256": "ba31b37e6fb4ffb9c08e7f5c240f707c95d53d1d259e5f6f23f141516a61d15f", "case": "artifacts/0ba58efa1499ae1de05f21f80e8fa33c67abbc4d100e7aeb635f7599de8d2603/case.json", "inputs": "artifacts/0ba58efa1499ae1de05f21f80e8fa33c67abbc4d100e7aeb635f7599de8d2603/inputs.npz", "manifest": "artifacts/0ba58efa1499ae1de05f21f80e8fa33c67abbc4d100e7aeb635f7599de8d2603/manifest.json", "model": "artifacts/0ba58efa1499ae1de05f21f80e8fa33c67abbc4d100e7aeb635f7599de8d2603/model.onnx", "outputs": "observations/d61707df367e219f7fc55d4c4333eb4982378e1e357ebdd9de83cea046cb1ee2/outputs.npz" }, "candidate_group": "53b6cb53a27ab9cf051b08a7a13916aaac3005d353ddfb43f3e2cef0c88ef097", "case_id": "0ba58efa1499ae1de05f21f80e8fa33c67abbc4d100e7aeb635f7599de8d2603", "category": "masking", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "backend": "reference", "name": "probability_range", "passed": true }, { "backend": "reference", "name": "masked_entries_exact_zero", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 0.000244140625, "max_budget_ratio": 0.04438920454545455, "max_rel": 0.000244140625, "max_ulp": 2199023255552, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 8, 17 ], "backend": "reference", "elements": 136, "expected_dtype": "float16", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "backend": "ort", "name": "probability_range", "passed": true }, { "backend": "ort", "name": "masked_entries_exact_zero", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 0.000244140625, "max_budget_ratio": 0.04438920454545455, "max_rel": 0.000244140625, "max_ulp": 2199023255552, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 8, 17 ], "backend": "ort", "elements": 136, "expected_dtype": "float16", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } } ], "comparison": { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, "comparison_key": "9e080f1ca7a1de532e71156c5472a07848d1d7296b4e340b7263cdd499a80c15", "contract": "masked_probability", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "mask": { "dtype": "bool", "shape": [ 4, 17 ] }, "x": { "dtype": "float16", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "MaskedSoftmax/constant/float16/4x17/seed-20260906", "observation_id": "d61707df367e219f7fc55d4c4333eb4982378e1e357ebdd9de83cea046cb1ee2", "operation": "MaskedSoftmax", "operators": [ "Div", "Equal", "Exp", "ReduceMax", "ReduceSum", "Sub", "Where" ], "opset": 19, "optimization": "all", "output_hash": "06f489e9057c8d194f115ba4208334258621a407cd815c66be3c373996ee3201", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.0085, "p95_ms": 0.0096498, "repeats": 7, "samples_ms": [ 0.00975, 0.009416, 0.008958, 0.0085, 0.007834, 0.00775, 0.00775 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.049542, "p95_ms": 0.052908, "repeats": 7, "samples_ms": [ 0.053583, 0.051333, 0.050333, 0.0495, 0.048375, 0.048792, 0.049542 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 0.0005, "rtol": 0.005 }, "variant": "constant", "warnings": [ { "backend": "reference", "message": "underflow encountered in exp" } ] }, { "artifacts": { "bundle": "observations/2a3ae6cf50a7d50fbc49048dca90bb827c349256b19cfad6b96df17fbfe8eaa8/reproducer.zip", "bundle_sha256": "5f616b1293e3011c015202672aa2dc85e0ea014251506588fd54189e003cb8a7", "case": "artifacts/0ba58efa1499ae1de05f21f80e8fa33c67abbc4d100e7aeb635f7599de8d2603/case.json", "inputs": "artifacts/0ba58efa1499ae1de05f21f80e8fa33c67abbc4d100e7aeb635f7599de8d2603/inputs.npz", "manifest": "artifacts/0ba58efa1499ae1de05f21f80e8fa33c67abbc4d100e7aeb635f7599de8d2603/manifest.json", "model": "artifacts/0ba58efa1499ae1de05f21f80e8fa33c67abbc4d100e7aeb635f7599de8d2603/model.onnx", "outputs": "observations/2a3ae6cf50a7d50fbc49048dca90bb827c349256b19cfad6b96df17fbfe8eaa8/outputs.npz" }, "candidate_group": "53b6cb53a27ab9cf051b08a7a13916aaac3005d353ddfb43f3e2cef0c88ef097", "case_id": "0ba58efa1499ae1de05f21f80e8fa33c67abbc4d100e7aeb635f7599de8d2603", "category": "masking", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "backend": "reference", "name": "probability_range", "passed": true }, { "backend": "reference", "name": "masked_entries_exact_zero", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 0.000244140625, "max_budget_ratio": 0.04438920454545455, "max_rel": 0.000244140625, "max_ulp": 2199023255552, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 8, 17 ], "backend": "reference", "elements": 136, "expected_dtype": "float16", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "backend": "ort", "name": "probability_range", "passed": true }, { "backend": "ort", "name": "masked_entries_exact_zero", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 0.000244140625, "max_budget_ratio": 0.04438920454545455, "max_rel": 0.000244140625, "max_ulp": 2199023255552, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 8, 17 ], "backend": "ort", "elements": 136, "expected_dtype": "float16", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } } ], "comparison": { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, "comparison_key": "1c95ef0966a33d8d121444cdf6d31bb4fd72f6af72a179238ae4286df4d5332f", "contract": "masked_probability", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "mask": { "dtype": "bool", "shape": [ 4, 17 ] }, "x": { "dtype": "float16", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "MaskedSoftmax/constant/float16/4x17/seed-20260906", "observation_id": "2a3ae6cf50a7d50fbc49048dca90bb827c349256b19cfad6b96df17fbfe8eaa8", "operation": "MaskedSoftmax", "operators": [ "Div", "Equal", "Exp", "ReduceMax", "ReduceSum", "Sub", "Where" ], "opset": 19, "optimization": "disabled", "output_hash": "06f489e9057c8d194f115ba4208334258621a407cd815c66be3c373996ee3201", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.008125, "p95_ms": 0.009425, "repeats": 7, "samples_ms": [ 0.0095, 0.00925, 0.008458, 0.008125, 0.008, 0.007917, 0.008084 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.0495, "p95_ms": 0.0501162, "repeats": 7, "samples_ms": [ 0.05, 0.048833, 0.04975, 0.0495, 0.048583, 0.048917, 0.050166 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 0.0005, "rtol": 0.005 }, "variant": "constant", "warnings": [ { "backend": "reference", "message": "underflow encountered in exp" } ] }, { "artifacts": { "bundle": "observations/3fc92c6bc972fb6d19c0c0e692fceab9facac347bdce19b27904aa52eeac8f81/reproducer.zip", "bundle_sha256": "316018dde332fe1297483016ba8e26c88fe800547433c82c50e5e82672345cab", "case": "artifacts/25aa1b51d3743e4c382be6015ea37a4361bc86611fb4f02417e55eb7e648ad7e/case.json", "inputs": "artifacts/25aa1b51d3743e4c382be6015ea37a4361bc86611fb4f02417e55eb7e648ad7e/inputs.npz", "manifest": "artifacts/25aa1b51d3743e4c382be6015ea37a4361bc86611fb4f02417e55eb7e648ad7e/manifest.json", "model": "artifacts/25aa1b51d3743e4c382be6015ea37a4361bc86611fb4f02417e55eb7e648ad7e/model.onnx", "outputs": "observations/3fc92c6bc972fb6d19c0c0e692fceab9facac347bdce19b27904aa52eeac8f81/outputs.npz" }, "candidate_group": "f42328a850dc72b630f90cdabb5636a87a1f345de4df54e0891eb6bfbf28b172", "case_id": "25aa1b51d3743e4c382be6015ea37a4361bc86611fb4f02417e55eb7e648ad7e", "category": "masking", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "backend": "reference", "name": "probability_range", "passed": true }, { "backend": "reference", "name": "masked_entries_exact_zero", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 5.21540641784668e-08, "max_budget_ratio": 0.004741278561678799, "max_rel": 5.21540641784668e-08, "max_ulp": 234881024, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 17 ], "backend": "reference", "elements": 136, "expected_dtype": "float32", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "backend": "ort", "name": "probability_range", "passed": true }, { "backend": "ort", "name": "masked_entries_exact_zero", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 5.21540641784668e-08, "max_budget_ratio": 0.004741278561678799, "max_rel": 5.21540641784668e-08, "max_ulp": 234881024, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 17 ], "backend": "ort", "elements": 136, "expected_dtype": "float32", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } } ], "comparison": { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, "comparison_key": "e626712f0ff29544a5ab481bacebd838fc027f790686995590408b333b40540e", "contract": "masked_probability", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "mask": { "dtype": "bool", "shape": [ 4, 17 ] }, "x": { "dtype": "float32", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "MaskedSoftmax/constant/float32/4x17/seed-20260906", "observation_id": "3fc92c6bc972fb6d19c0c0e692fceab9facac347bdce19b27904aa52eeac8f81", "operation": "MaskedSoftmax", "operators": [ "Div", "Equal", "Exp", "ReduceMax", "ReduceSum", "Sub", "Where" ], "opset": 19, "optimization": "all", "output_hash": "4bc4ce81dc87fe8d3a4776075b229a0f4ddcf944ec2a6e09406eb0aeec6bfeaf", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.007416, "p95_ms": 0.0080161, "repeats": 7, "samples_ms": [ 0.008041, 0.007958, 0.00725, 0.007416, 0.0075, 0.007333, 0.007375 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.048541, "p95_ms": 0.0508411, "repeats": 7, "samples_ms": [ 0.051166, 0.050083, 0.048958, 0.048541, 0.047584, 0.048, 0.047458 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-06, "rtol": 1e-05 }, "variant": "constant", "warnings": [ { "backend": "reference", "message": "underflow encountered in exp" } ] }, { "artifacts": { "bundle": "observations/a78a0276fe9dc48efb877c331da13569fd47c84146050f561ce46debdcf159ba/reproducer.zip", "bundle_sha256": "ded94d4d435915f31ad86876b64988715de20ef4fb6d8aab22bfe9982283c3ae", "case": "artifacts/25aa1b51d3743e4c382be6015ea37a4361bc86611fb4f02417e55eb7e648ad7e/case.json", "inputs": "artifacts/25aa1b51d3743e4c382be6015ea37a4361bc86611fb4f02417e55eb7e648ad7e/inputs.npz", "manifest": "artifacts/25aa1b51d3743e4c382be6015ea37a4361bc86611fb4f02417e55eb7e648ad7e/manifest.json", "model": "artifacts/25aa1b51d3743e4c382be6015ea37a4361bc86611fb4f02417e55eb7e648ad7e/model.onnx", "outputs": "observations/a78a0276fe9dc48efb877c331da13569fd47c84146050f561ce46debdcf159ba/outputs.npz" }, "candidate_group": "f42328a850dc72b630f90cdabb5636a87a1f345de4df54e0891eb6bfbf28b172", "case_id": "25aa1b51d3743e4c382be6015ea37a4361bc86611fb4f02417e55eb7e648ad7e", "category": "masking", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "backend": "reference", "name": "probability_range", "passed": true }, { "backend": "reference", "name": "masked_entries_exact_zero", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 5.21540641784668e-08, "max_budget_ratio": 0.004741278561678799, "max_rel": 5.21540641784668e-08, "max_ulp": 234881024, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 17 ], "backend": "reference", "elements": 136, "expected_dtype": "float32", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "backend": "ort", "name": "probability_range", "passed": true }, { "backend": "ort", "name": "masked_entries_exact_zero", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 5.21540641784668e-08, "max_budget_ratio": 0.004741278561678799, "max_rel": 5.21540641784668e-08, "max_ulp": 234881024, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 17 ], "backend": "ort", "elements": 136, "expected_dtype": "float32", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } } ], "comparison": { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, "comparison_key": "4cda5e6fb6d401a1a3bc194122a6e6a1120132c51ac9428db1aa93bd45743ee3", "contract": "masked_probability", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "mask": { "dtype": "bool", "shape": [ 4, 17 ] }, "x": { "dtype": "float32", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "MaskedSoftmax/constant/float32/4x17/seed-20260906", "observation_id": "a78a0276fe9dc48efb877c331da13569fd47c84146050f561ce46debdcf159ba", "operation": "MaskedSoftmax", "operators": [ "Div", "Equal", "Exp", "ReduceMax", "ReduceSum", "Sub", "Where" ], "opset": 19, "optimization": "disabled", "output_hash": "4bc4ce81dc87fe8d3a4776075b229a0f4ddcf944ec2a6e09406eb0aeec6bfeaf", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.00725, "p95_ms": 0.0076001, "repeats": 7, "samples_ms": [ 0.007542, 0.007625, 0.00725, 0.007125, 0.007209, 0.007291, 0.007 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.049125, "p95_ms": 0.0509456, "repeats": 7, "samples_ms": [ 0.048625, 0.049125, 0.048542, 0.050625, 0.051083, 0.048917, 0.049167 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-06, "rtol": 1e-05 }, "variant": "constant", "warnings": [ { "backend": "reference", "message": "underflow encountered in exp" } ] }, { "artifacts": { "bundle": "observations/af35107512b3808dc91c38ec1f8e9846c5fe74353fd0fe16aca9dcaa23e8c748/reproducer.zip", "bundle_sha256": "37c47467e688382c489424b09290dbc6fdc6893979716927e927a2793ee6a704", "case": "artifacts/f4a587688c37ae97485e0e118a75458541c0e1cde88837554e90545def7b92ff/case.json", "inputs": "artifacts/f4a587688c37ae97485e0e118a75458541c0e1cde88837554e90545def7b92ff/inputs.npz", "manifest": "artifacts/f4a587688c37ae97485e0e118a75458541c0e1cde88837554e90545def7b92ff/manifest.json", "model": "artifacts/f4a587688c37ae97485e0e118a75458541c0e1cde88837554e90545def7b92ff/model.onnx", "outputs": "observations/af35107512b3808dc91c38ec1f8e9846c5fe74353fd0fe16aca9dcaa23e8c748/outputs.npz" }, "candidate_group": "67f30326e18048fc7358820b0ba1d9f62a0326dee6616c7c568832ba3dfc9fd1", "case_id": "f4a587688c37ae97485e0e118a75458541c0e1cde88837554e90545def7b92ff", "category": "masking", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "backend": "reference", "name": "probability_range", "passed": true }, { "backend": "reference", "name": "masked_entries_exact_zero", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 8, 17 ], "backend": "reference", "elements": 136, "expected_dtype": "float64", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "backend": "ort", "name": "probability_range", "passed": true }, { "backend": "ort", "name": "masked_entries_exact_zero", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 8, 17 ], "backend": "ort", "elements": 136, "expected_dtype": "float64", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } } ], "comparison": { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, "comparison_key": "719ac28d3fdf45e6ef8aad1bb3da2d8c5a601195616940cc858b3f169ff5b0a0", "contract": "masked_probability", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "mask": { "dtype": "bool", "shape": [ 4, 17 ] }, "x": { "dtype": "float64", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "MaskedSoftmax/constant/float64/4x17/seed-20260906", "observation_id": "af35107512b3808dc91c38ec1f8e9846c5fe74353fd0fe16aca9dcaa23e8c748", "operation": "MaskedSoftmax", "operators": [ "Div", "Equal", "Exp", "ReduceMax", "ReduceSum", "Sub", "Where" ], "opset": 19, "optimization": "all", "output_hash": "775cdea543f03539a2e178a72f548874fac3be5120c2464bca72100338b32926", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.007625, "p95_ms": 0.0080293, "repeats": 7, "samples_ms": [ 0.008167, 0.007708, 0.007667, 0.007375, 0.007625, 0.007583, 0.007084 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.047375, "p95_ms": 0.0494373, "repeats": 7, "samples_ms": [ 0.049291, 0.0495, 0.048, 0.047375, 0.046334, 0.046167, 0.047042 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-13, "rtol": 1e-12 }, "variant": "constant", "warnings": [ { "backend": "reference", "message": "underflow encountered in exp" } ] }, { "artifacts": { "bundle": "observations/810e15ac10af3a486fc4750f554177a9c4226346b7ba8ed45f62335bce2f70ff/reproducer.zip", "bundle_sha256": "35dd2b69b44d5340891777dd5471682f3493f25ab826c16d6bc44323d6b08a9a", "case": "artifacts/f4a587688c37ae97485e0e118a75458541c0e1cde88837554e90545def7b92ff/case.json", "inputs": "artifacts/f4a587688c37ae97485e0e118a75458541c0e1cde88837554e90545def7b92ff/inputs.npz", "manifest": "artifacts/f4a587688c37ae97485e0e118a75458541c0e1cde88837554e90545def7b92ff/manifest.json", "model": "artifacts/f4a587688c37ae97485e0e118a75458541c0e1cde88837554e90545def7b92ff/model.onnx", "outputs": "observations/810e15ac10af3a486fc4750f554177a9c4226346b7ba8ed45f62335bce2f70ff/outputs.npz" }, "candidate_group": "67f30326e18048fc7358820b0ba1d9f62a0326dee6616c7c568832ba3dfc9fd1", "case_id": "f4a587688c37ae97485e0e118a75458541c0e1cde88837554e90545def7b92ff", "category": "masking", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "backend": "reference", "name": "probability_range", "passed": true }, { "backend": "reference", "name": "masked_entries_exact_zero", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 8, 17 ], "backend": "reference", "elements": 136, "expected_dtype": "float64", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "backend": "ort", "name": "probability_range", "passed": true }, { "backend": "ort", "name": "masked_entries_exact_zero", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 8, 17 ], "backend": "ort", "elements": 136, "expected_dtype": "float64", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } } ], "comparison": { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, "comparison_key": "c5a3de11526b283ceb78c80c67f92ddffacb61b40db50df3097995f86f466331", "contract": "masked_probability", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "mask": { "dtype": "bool", "shape": [ 4, 17 ] }, "x": { "dtype": "float64", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "MaskedSoftmax/constant/float64/4x17/seed-20260906", "observation_id": "810e15ac10af3a486fc4750f554177a9c4226346b7ba8ed45f62335bce2f70ff", "operation": "MaskedSoftmax", "operators": [ "Div", "Equal", "Exp", "ReduceMax", "ReduceSum", "Sub", "Where" ], "opset": 19, "optimization": "disabled", "output_hash": "775cdea543f03539a2e178a72f548874fac3be5120c2464bca72100338b32926", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.007583, "p95_ms": 0.0084249, "repeats": 7, "samples_ms": [ 0.008625, 0.007958, 0.007583, 0.00725, 0.007917, 0.007291, 0.00725 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.04975, "p95_ms": 0.0508375, "repeats": 7, "samples_ms": [ 0.05125, 0.049875, 0.049875, 0.04975, 0.048125, 0.049209, 0.048875 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-13, "rtol": 1e-12 }, "variant": "constant", "warnings": [ { "backend": "reference", "message": "underflow encountered in exp" } ] }, { "artifacts": { "bundle": "observations/e029c45d60f81a98e03f0c391a184fd03facd3ee20834e8e8eac4a6edcd75e63/reproducer.zip", "bundle_sha256": "a7c90425df5570ce41d8709998a514ba5cb340caa410d1d31d12e0fd34b2ee64", "case": "artifacts/8a2a89b1db2dd8dfd63ec83bb90c4fed6516c3ee1472f8d3956186340ea45b4d/case.json", "inputs": "artifacts/8a2a89b1db2dd8dfd63ec83bb90c4fed6516c3ee1472f8d3956186340ea45b4d/inputs.npz", "manifest": "artifacts/8a2a89b1db2dd8dfd63ec83bb90c4fed6516c3ee1472f8d3956186340ea45b4d/manifest.json", "model": "artifacts/8a2a89b1db2dd8dfd63ec83bb90c4fed6516c3ee1472f8d3956186340ea45b4d/model.onnx", "outputs": "observations/e029c45d60f81a98e03f0c391a184fd03facd3ee20834e8e8eac4a6edcd75e63/outputs.npz" }, "candidate_group": "d3866a6f1b2499d9ff4edd74a945b4c7e439ed124e61ec968aae18b048e736e3", "case_id": "8a2a89b1db2dd8dfd63ec83bb90c4fed6516c3ee1472f8d3956186340ea45b4d", "category": "masking", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "backend": "reference", "name": "probability_range", "passed": true }, { "backend": "reference", "name": "masked_entries_exact_zero", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 0.0003662109375, "max_budget_ratio": 0.06658380681818182, "max_rel": 0.0003662109375, "max_ulp": 1649267441664, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 8, 17 ], "backend": "reference", "elements": 136, "expected_dtype": "float16", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "backend": "ort", "name": "probability_range", "passed": true }, { "backend": "ort", "name": "masked_entries_exact_zero", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 6.103515625e-05, "max_budget_ratio": 0.011097301136363638, "max_rel": 6.103515625e-05, "max_ulp": 549755813888, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 8, 17 ], "backend": "ort", "elements": 136, "expected_dtype": "float16", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } } ], "comparison": { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 6.103515625e-05, "max_budget_ratio": 0.07456009543692216, "max_rel": 0.0009578544061302681, "max_ulp": 1, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, "comparison_key": "fc1e66284780b7aec0466af24e285c38d17f3fa82e4530ce90ac5ca8b398a5a4", "contract": "masked_probability", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "mask": { "dtype": "bool", "shape": [ 4, 17 ] }, "x": { "dtype": "float16", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "MaskedSoftmax/large_offset/float16/4x17/seed-20260906", "observation_id": "e029c45d60f81a98e03f0c391a184fd03facd3ee20834e8e8eac4a6edcd75e63", "operation": "MaskedSoftmax", "operators": [ "Div", "Equal", "Exp", "ReduceMax", "ReduceSum", "Sub", "Where" ], "opset": 19, "optimization": "all", "output_hash": "3c2b8fdb6ca5501614ed4ea80cc61e181c37d16d60ea7c756f5345f096b07d02", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.008458, "p95_ms": 0.0087545, "repeats": 7, "samples_ms": [ 0.008667, 0.008792, 0.008667, 0.008458, 0.008375, 0.008084, 0.007875 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.051417, "p95_ms": 0.09292489999999996, "repeats": 7, "samples_ms": [ 0.055708, 0.051959, 0.051375, 0.051375, 0.051417, 0.050416, 0.108875 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 0.0005, "rtol": 0.005 }, "variant": "large_offset", "warnings": [ { "backend": "reference", "message": "overflow encountered in subtract" } ] }, { "artifacts": { "bundle": "observations/879496276c0c1e2a8602dc42d05335cce944bdff025d43f196e9d0a70e35dc5e/reproducer.zip", "bundle_sha256": "675ead1e23e28711c5b5146d8bf49a25fef44d9a1ccabf001ca7c92973bd9bae", "case": "artifacts/8a2a89b1db2dd8dfd63ec83bb90c4fed6516c3ee1472f8d3956186340ea45b4d/case.json", "inputs": "artifacts/8a2a89b1db2dd8dfd63ec83bb90c4fed6516c3ee1472f8d3956186340ea45b4d/inputs.npz", "manifest": "artifacts/8a2a89b1db2dd8dfd63ec83bb90c4fed6516c3ee1472f8d3956186340ea45b4d/manifest.json", "model": "artifacts/8a2a89b1db2dd8dfd63ec83bb90c4fed6516c3ee1472f8d3956186340ea45b4d/model.onnx", "outputs": "observations/879496276c0c1e2a8602dc42d05335cce944bdff025d43f196e9d0a70e35dc5e/outputs.npz" }, "candidate_group": "d3866a6f1b2499d9ff4edd74a945b4c7e439ed124e61ec968aae18b048e736e3", "case_id": "8a2a89b1db2dd8dfd63ec83bb90c4fed6516c3ee1472f8d3956186340ea45b4d", "category": "masking", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "backend": "reference", "name": "probability_range", "passed": true }, { "backend": "reference", "name": "masked_entries_exact_zero", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 0.0003662109375, "max_budget_ratio": 0.06658380681818182, "max_rel": 0.0003662109375, "max_ulp": 1649267441664, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 8, 17 ], "backend": "reference", "elements": 136, "expected_dtype": "float16", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "backend": "ort", "name": "probability_range", "passed": true }, { "backend": "ort", "name": "masked_entries_exact_zero", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 6.103515625e-05, "max_budget_ratio": 0.011097301136363638, "max_rel": 6.103515625e-05, "max_ulp": 549755813888, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 8, 17 ], "backend": "ort", "elements": 136, "expected_dtype": "float16", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } } ], "comparison": { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 6.103515625e-05, "max_budget_ratio": 0.07456009543692216, "max_rel": 0.0009578544061302681, "max_ulp": 1, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, "comparison_key": "4f96e755bf8431efa2e7bfad2a9246d16774770cf8c87fc0f7d8ed039164d02e", "contract": "masked_probability", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "mask": { "dtype": "bool", "shape": [ 4, 17 ] }, "x": { "dtype": "float16", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "MaskedSoftmax/large_offset/float16/4x17/seed-20260906", "observation_id": "879496276c0c1e2a8602dc42d05335cce944bdff025d43f196e9d0a70e35dc5e", "operation": "MaskedSoftmax", "operators": [ "Div", "Equal", "Exp", "ReduceMax", "ReduceSum", "Sub", "Where" ], "opset": 19, "optimization": "disabled", "output_hash": "3c2b8fdb6ca5501614ed4ea80cc61e181c37d16d60ea7c756f5345f096b07d02", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.008375, "p95_ms": 0.0091124, "repeats": 7, "samples_ms": [ 0.009125, 0.009083, 0.008583, 0.008375, 0.008334, 0.008, 0.007833 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.050208, "p95_ms": 0.051875, "repeats": 7, "samples_ms": [ 0.05225, 0.050458, 0.050041, 0.051, 0.049417, 0.050208, 0.049375 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 0.0005, "rtol": 0.005 }, "variant": "large_offset", "warnings": [ { "backend": "reference", "message": "overflow encountered in subtract" } ] }, { "artifacts": { "bundle": "observations/d26c012503fe0c0d39b2073e9a885d2a098bb0785770dfe4b1a1b1a033bafe77/reproducer.zip", "bundle_sha256": "f46621145c0982dc8a2d1beb19848854eb03b5612d9f24680e22387bbd0d68a1", "case": "artifacts/1290c2e84d1ef847579fe32f733195d67fc05e3341f874f768726deb31d4a268/case.json", "inputs": "artifacts/1290c2e84d1ef847579fe32f733195d67fc05e3341f874f768726deb31d4a268/inputs.npz", "manifest": "artifacts/1290c2e84d1ef847579fe32f733195d67fc05e3341f874f768726deb31d4a268/manifest.json", "model": "artifacts/1290c2e84d1ef847579fe32f733195d67fc05e3341f874f768726deb31d4a268/model.onnx", "outputs": "observations/d26c012503fe0c0d39b2073e9a885d2a098bb0785770dfe4b1a1b1a033bafe77/outputs.npz" }, "candidate_group": "446331bf7d45b9984fd425cc6e46f9298df57374e27ee38d673f758b6f7111a5", "case_id": "1290c2e84d1ef847579fe32f733195d67fc05e3341f874f768726deb31d4a268", "category": "masking", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "backend": "reference", "name": "probability_range", "passed": true }, { "backend": "reference", "name": "masked_entries_exact_zero", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 6.705522537231445e-08, "max_budget_ratio": 0.0060959295793013134, "max_rel": 6.705522537231445e-08, "max_ulp": 603979776, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 17 ], "backend": "reference", "elements": 136, "expected_dtype": "float32", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "backend": "ort", "name": "probability_range", "passed": true }, { "backend": "ort", "name": "masked_entries_exact_zero", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 2.9802322387695312e-08, "max_budget_ratio": 0.002709302035245028, "max_rel": 2.9802322387695312e-08, "max_ulp": 134217728, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 1.4901161193847656e-08, "max_budget_ratio": 0.00754191813181499, "max_rel": 1.527104579175791e-07, "max_ulp": 2, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 1.4901161193847656e-08, "max_budget_ratio": 0.00754191813181499, "max_rel": 1.527104579175791e-07, "max_ulp": 2, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 17 ], "backend": "ort", "elements": 136, "expected_dtype": "float32", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } } ], "comparison": { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 1.4901161193847656e-08, "max_budget_ratio": 0.007541918700620325, "max_rel": 1.5271048123806662e-07, "max_ulp": 2, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, "comparison_key": "51de4ddf11cf4b1865b9e9c0875832af145b6dd1bfd2b40a4549492f33fa77b5", "contract": "masked_probability", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "mask": { "dtype": "bool", "shape": [ 4, 17 ] }, "x": { "dtype": "float32", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "MaskedSoftmax/large_offset/float32/4x17/seed-20260906", "observation_id": "d26c012503fe0c0d39b2073e9a885d2a098bb0785770dfe4b1a1b1a033bafe77", "operation": "MaskedSoftmax", "operators": [ "Div", "Equal", "Exp", "ReduceMax", "ReduceSum", "Sub", "Where" ], "opset": 19, "optimization": "all", "output_hash": "bd1d2be8cd685cd3c64e02b02d556a4daa0b2f5888e26e2e1320faa1100f251e", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.007875, "p95_ms": 0.008167, "repeats": 7, "samples_ms": [ 0.008167, 0.008167, 0.007542, 0.007875, 0.007916, 0.007584, 0.007667 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.050209, "p95_ms": 0.0518787, "repeats": 7, "samples_ms": [ 0.052416, 0.050625, 0.050209, 0.050083, 0.050375, 0.0495, 0.049958 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-06, "rtol": 1e-05 }, "variant": "large_offset", "warnings": [ { "backend": "reference", "message": "underflow encountered in exp" } ] }, { "artifacts": { "bundle": "observations/f6041265300f2563bc5a7376d7c6a1e1aa08f3ae44dec9daa844b52e14e878bb/reproducer.zip", "bundle_sha256": "c93c98687e2259bddcfba10341520d3ad7f4c93b8983fd73c7286c6a26b70d06", "case": "artifacts/1290c2e84d1ef847579fe32f733195d67fc05e3341f874f768726deb31d4a268/case.json", "inputs": "artifacts/1290c2e84d1ef847579fe32f733195d67fc05e3341f874f768726deb31d4a268/inputs.npz", "manifest": "artifacts/1290c2e84d1ef847579fe32f733195d67fc05e3341f874f768726deb31d4a268/manifest.json", "model": "artifacts/1290c2e84d1ef847579fe32f733195d67fc05e3341f874f768726deb31d4a268/model.onnx", "outputs": "observations/f6041265300f2563bc5a7376d7c6a1e1aa08f3ae44dec9daa844b52e14e878bb/outputs.npz" }, "candidate_group": "446331bf7d45b9984fd425cc6e46f9298df57374e27ee38d673f758b6f7111a5", "case_id": "1290c2e84d1ef847579fe32f733195d67fc05e3341f874f768726deb31d4a268", "category": "masking", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "backend": "reference", "name": "probability_range", "passed": true }, { "backend": "reference", "name": "masked_entries_exact_zero", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 6.705522537231445e-08, "max_budget_ratio": 0.0060959295793013134, "max_rel": 6.705522537231445e-08, "max_ulp": 603979776, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 17 ], "backend": "reference", "elements": 136, "expected_dtype": "float32", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "backend": "ort", "name": "probability_range", "passed": true }, { "backend": "ort", "name": "masked_entries_exact_zero", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 2.9802322387695312e-08, "max_budget_ratio": 0.002709302035245028, "max_rel": 2.9802322387695312e-08, "max_ulp": 134217728, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 1.4901161193847656e-08, "max_budget_ratio": 0.00754191813181499, "max_rel": 1.527104579175791e-07, "max_ulp": 2, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 1.4901161193847656e-08, "max_budget_ratio": 0.00754191813181499, "max_rel": 1.527104579175791e-07, "max_ulp": 2, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 17 ], "backend": "ort", "elements": 136, "expected_dtype": "float32", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } } ], "comparison": { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 1.4901161193847656e-08, "max_budget_ratio": 0.007541918700620325, "max_rel": 1.5271048123806662e-07, "max_ulp": 2, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, "comparison_key": "3243425fc849234d61b45da4c0e8ff391e21d80f2267800480bae28c60aea0d9", "contract": "masked_probability", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "mask": { "dtype": "bool", "shape": [ 4, 17 ] }, "x": { "dtype": "float32", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "MaskedSoftmax/large_offset/float32/4x17/seed-20260906", "observation_id": "f6041265300f2563bc5a7376d7c6a1e1aa08f3ae44dec9daa844b52e14e878bb", "operation": "MaskedSoftmax", "operators": [ "Div", "Equal", "Exp", "ReduceMax", "ReduceSum", "Sub", "Where" ], "opset": 19, "optimization": "disabled", "output_hash": "bd1d2be8cd685cd3c64e02b02d556a4daa0b2f5888e26e2e1320faa1100f251e", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.007333, "p95_ms": 0.0080124, "repeats": 7, "samples_ms": [ 0.007458, 0.00825, 0.007417, 0.007208, 0.007333, 0.00725, 0.007041 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.049917, "p95_ms": 0.0522708, "repeats": 7, "samples_ms": [ 0.050959, 0.050708, 0.049917, 0.052833, 0.049541, 0.049292, 0.048958 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-06, "rtol": 1e-05 }, "variant": "large_offset", "warnings": [ { "backend": "reference", "message": "underflow encountered in exp" } ] }, { "artifacts": { "bundle": "observations/85e85eda260f6466845475b432f4f6483f303b09bae1f330cc73efcbfdcb9c5b/reproducer.zip", "bundle_sha256": "448d62c99dd18c72a19edd584eb68ea6acf4f06dd1043f16ec72e345ec4bf6b3", "case": "artifacts/0300afd8ba7382af3b3cee23ea69ce80029da6d4d56aaa48999861b5983e4adb/case.json", "inputs": "artifacts/0300afd8ba7382af3b3cee23ea69ce80029da6d4d56aaa48999861b5983e4adb/inputs.npz", "manifest": "artifacts/0300afd8ba7382af3b3cee23ea69ce80029da6d4d56aaa48999861b5983e4adb/manifest.json", "model": "artifacts/0300afd8ba7382af3b3cee23ea69ce80029da6d4d56aaa48999861b5983e4adb/model.onnx", "outputs": "observations/85e85eda260f6466845475b432f4f6483f303b09bae1f330cc73efcbfdcb9c5b/outputs.npz" }, "candidate_group": "982f262bccc9263bfabaa80693fc4b3f94474fb019b95276ffa7de3b15312a8f", "case_id": "0300afd8ba7382af3b3cee23ea69ce80029da6d4d56aaa48999861b5983e4adb", "category": "masking", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "backend": "reference", "name": "probability_range", "passed": true }, { "backend": "reference", "name": "masked_entries_exact_zero", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 2.220446049250313e-16, "max_budget_ratio": 0.00020185873175002848, "max_rel": 2.220446049250313e-16, "max_ulp": 1, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 8, 17 ], "backend": "reference", "elements": 136, "expected_dtype": "float64", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "backend": "ort", "name": "probability_range", "passed": true }, { "backend": "ort", "name": "masked_entries_exact_zero", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 2.220446049250313e-16, "max_budget_ratio": 0.00020185873175002848, "max_rel": 2.220446049250313e-16, "max_ulp": 1, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 8, 17 ], "backend": "ort", "elements": 136, "expected_dtype": "float64", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } } ], "comparison": { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 6.938893903907228e-18, "max_budget_ratio": 4.5366813721151406e-05, "max_rel": 1.3104398674273416e-16, "max_ulp": 1, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, "comparison_key": "d6a6a913d888aa0ab970099b3238aa05457df0ba45d7924d48ee5af072c68b25", "contract": "masked_probability", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "mask": { "dtype": "bool", "shape": [ 4, 17 ] }, "x": { "dtype": "float64", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "MaskedSoftmax/large_offset/float64/4x17/seed-20260906", "observation_id": "85e85eda260f6466845475b432f4f6483f303b09bae1f330cc73efcbfdcb9c5b", "operation": "MaskedSoftmax", "operators": [ "Div", "Equal", "Exp", "ReduceMax", "ReduceSum", "Sub", "Where" ], "opset": 19, "optimization": "all", "output_hash": "1062fbd3e3aef050ee101cdb4f7272f288471421f2c310102473fc284b8ede79", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.007708, "p95_ms": 0.0082419, "repeats": 7, "samples_ms": [ 0.008084, 0.008292, 0.008125, 0.007459, 0.007708, 0.007625, 0.00725 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.048708, "p95_ms": 0.0490545, "repeats": 7, "samples_ms": [ 0.044666, 0.048792, 0.04875, 0.0485, 0.048708, 0.049167, 0.047875 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-13, "rtol": 1e-12 }, "variant": "large_offset", "warnings": [ { "backend": "reference", "message": "underflow encountered in exp" } ] }, { "artifacts": { "bundle": "observations/afaf0c4db10ea8de5087d7e67915767eae52b62c9cd98dffce5cbeddefea0425/reproducer.zip", "bundle_sha256": "2a48f49f83ff35b5ac6e843a0a30756fafe9ee0c0849822cf8247bda8d48d7b7", "case": "artifacts/0300afd8ba7382af3b3cee23ea69ce80029da6d4d56aaa48999861b5983e4adb/case.json", "inputs": "artifacts/0300afd8ba7382af3b3cee23ea69ce80029da6d4d56aaa48999861b5983e4adb/inputs.npz", "manifest": "artifacts/0300afd8ba7382af3b3cee23ea69ce80029da6d4d56aaa48999861b5983e4adb/manifest.json", "model": "artifacts/0300afd8ba7382af3b3cee23ea69ce80029da6d4d56aaa48999861b5983e4adb/model.onnx", "outputs": "observations/afaf0c4db10ea8de5087d7e67915767eae52b62c9cd98dffce5cbeddefea0425/outputs.npz" }, "candidate_group": "982f262bccc9263bfabaa80693fc4b3f94474fb019b95276ffa7de3b15312a8f", "case_id": "0300afd8ba7382af3b3cee23ea69ce80029da6d4d56aaa48999861b5983e4adb", "category": "masking", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "backend": "reference", "name": "probability_range", "passed": true }, { "backend": "reference", "name": "masked_entries_exact_zero", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 2.220446049250313e-16, "max_budget_ratio": 0.00020185873175002848, "max_rel": 2.220446049250313e-16, "max_ulp": 1, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 8, 17 ], "backend": "reference", "elements": 136, "expected_dtype": "float64", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "backend": "ort", "name": "probability_range", "passed": true }, { "backend": "ort", "name": "masked_entries_exact_zero", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 2.220446049250313e-16, "max_budget_ratio": 0.00020185873175002848, "max_rel": 2.220446049250313e-16, "max_ulp": 1, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 8, 17 ], "backend": "ort", "elements": 136, "expected_dtype": "float64", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } } ], "comparison": { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 6.938893903907228e-18, "max_budget_ratio": 4.5366813721151406e-05, "max_rel": 1.3104398674273416e-16, "max_ulp": 1, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, "comparison_key": "f5be2559a93c3ac8f9d30f981ec5308c72b7896603d0e1c8c8f652f7fdc85848", "contract": "masked_probability", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "mask": { "dtype": "bool", "shape": [ 4, 17 ] }, "x": { "dtype": "float64", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "MaskedSoftmax/large_offset/float64/4x17/seed-20260906", "observation_id": "afaf0c4db10ea8de5087d7e67915767eae52b62c9cd98dffce5cbeddefea0425", "operation": "MaskedSoftmax", "operators": [ "Div", "Equal", "Exp", "ReduceMax", "ReduceSum", "Sub", "Where" ], "opset": 19, "optimization": "disabled", "output_hash": "1062fbd3e3aef050ee101cdb4f7272f288471421f2c310102473fc284b8ede79", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.007709, "p95_ms": 0.0083626, "repeats": 7, "samples_ms": [ 0.007917, 0.0085, 0.008042, 0.007709, 0.007666, 0.007667, 0.007166 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.048416, "p95_ms": 0.0490208, "repeats": 7, "samples_ms": [ 0.046167, 0.046667, 0.049208, 0.048416, 0.048, 0.048584, 0.0485 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-13, "rtol": 1e-12 }, "variant": "large_offset", "warnings": [ { "backend": "reference", "message": "underflow encountered in exp" } ] }, { "artifacts": { "bundle": "observations/bc81b012772e5ef5bf8267019eace0c9ff4a46e6eba66dd41f3c568298848b6a/reproducer.zip", "bundle_sha256": "3cd47697c0ad3ca0e79cf442b3d2d24c0dbd33de1df07f5e34cfddc7ac5d12f4", "case": "artifacts/8a2c36ebfc74c3e1d99850f0ef420fa444abbd3c3df1dc87fe850ade9dc86894/case.json", "inputs": "artifacts/8a2c36ebfc74c3e1d99850f0ef420fa444abbd3c3df1dc87fe850ade9dc86894/inputs.npz", "manifest": "artifacts/8a2c36ebfc74c3e1d99850f0ef420fa444abbd3c3df1dc87fe850ade9dc86894/manifest.json", "model": "artifacts/8a2c36ebfc74c3e1d99850f0ef420fa444abbd3c3df1dc87fe850ade9dc86894/model.onnx", "outputs": "observations/bc81b012772e5ef5bf8267019eace0c9ff4a46e6eba66dd41f3c568298848b6a/outputs.npz" }, "candidate_group": "c851069690d62d9f0995325727ab3a45782fa0f9e4926ff7fd81e6a345e079db", "case_id": "8a2c36ebfc74c3e1d99850f0ef420fa444abbd3c3df1dc87fe850ade9dc86894", "category": "masking", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "backend": "reference", "name": "probability_range", "passed": true }, { "backend": "reference", "name": "masked_entries_exact_zero", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 0.0003070831298828125, "max_budget_ratio": 0.05583329634232955, "max_rel": 0.0003070831298828125, "max_ulp": 1382979469312, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 8, 17 ], "backend": "reference", "elements": 136, "expected_dtype": "float16", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "backend": "ort", "name": "probability_range", "passed": true }, { "backend": "ort", "name": "masked_entries_exact_zero", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 5.340576171875e-05, "max_budget_ratio": 0.009710138494318182, "max_rel": 5.340576171875e-05, "max_ulp": 481036337152, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 8, 17 ], "backend": "ort", "elements": 136, "expected_dtype": "float16", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } } ], "comparison": { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.000244140625, "max_budget_ratio": 0.12303149606299213, "max_rel": 0.0013956734124214933, "max_ulp": 2, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, "comparison_key": "4d1262bc20629eee676c85ffc6047b064131d0f2686d61dae2b86523886274a2", "contract": "masked_probability", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "mask": { "dtype": "bool", "shape": [ 4, 17 ] }, "x": { "dtype": "float16", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "MaskedSoftmax/ordinary/float16/4x17/seed-20260906", "observation_id": "bc81b012772e5ef5bf8267019eace0c9ff4a46e6eba66dd41f3c568298848b6a", "operation": "MaskedSoftmax", "operators": [ "Div", "Equal", "Exp", "ReduceMax", "ReduceSum", "Sub", "Where" ], "opset": 19, "optimization": "all", "output_hash": "32a649a8692e9d97899af94746f67734e71d0f07cb228f4ecb2635ea9a0da2a7", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.008625, "p95_ms": 0.009342, "repeats": 7, "samples_ms": [ 0.009417, 0.009167, 0.008709, 0.008625, 0.007833, 0.008125, 0.007792 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.0495, "p95_ms": 0.05185, "repeats": 7, "samples_ms": [ 0.052, 0.0515, 0.050834, 0.0495, 0.049166, 0.0495, 0.048959 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 0.0005, "rtol": 0.005 }, "variant": "ordinary", "warnings": [ { "backend": "reference", "message": "underflow encountered in exp" } ] }, { "artifacts": { "bundle": "observations/df6807e762de8c0aa8e286829d98416b0da3c7d8474fb723cb984522edf69a12/reproducer.zip", "bundle_sha256": "7a686e54eabd563ceef8d447b40203451929ed2909057751e4230b3abf942e52", "case": "artifacts/8a2c36ebfc74c3e1d99850f0ef420fa444abbd3c3df1dc87fe850ade9dc86894/case.json", "inputs": "artifacts/8a2c36ebfc74c3e1d99850f0ef420fa444abbd3c3df1dc87fe850ade9dc86894/inputs.npz", "manifest": "artifacts/8a2c36ebfc74c3e1d99850f0ef420fa444abbd3c3df1dc87fe850ade9dc86894/manifest.json", "model": "artifacts/8a2c36ebfc74c3e1d99850f0ef420fa444abbd3c3df1dc87fe850ade9dc86894/model.onnx", "outputs": "observations/df6807e762de8c0aa8e286829d98416b0da3c7d8474fb723cb984522edf69a12/outputs.npz" }, "candidate_group": "c851069690d62d9f0995325727ab3a45782fa0f9e4926ff7fd81e6a345e079db", "case_id": "8a2c36ebfc74c3e1d99850f0ef420fa444abbd3c3df1dc87fe850ade9dc86894", "category": "masking", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "backend": "reference", "name": "probability_range", "passed": true }, { "backend": "reference", "name": "masked_entries_exact_zero", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 0.0003070831298828125, "max_budget_ratio": 0.05583329634232955, "max_rel": 0.0003070831298828125, "max_ulp": 1382979469312, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 8, 17 ], "backend": "reference", "elements": 136, "expected_dtype": "float16", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "backend": "ort", "name": "probability_range", "passed": true }, { "backend": "ort", "name": "masked_entries_exact_zero", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 5.340576171875e-05, "max_budget_ratio": 0.009710138494318182, "max_rel": 5.340576171875e-05, "max_ulp": 481036337152, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 8, 17 ], "backend": "ort", "elements": 136, "expected_dtype": "float16", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } } ], "comparison": { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.000244140625, "max_budget_ratio": 0.12303149606299213, "max_rel": 0.0013956734124214933, "max_ulp": 2, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, "comparison_key": "845dd11844774bfc2ce00ffe64f14a9278938a8178bbf222fbcd1de09b486c5e", "contract": "masked_probability", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "mask": { "dtype": "bool", "shape": [ 4, 17 ] }, "x": { "dtype": "float16", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "MaskedSoftmax/ordinary/float16/4x17/seed-20260906", "observation_id": "df6807e762de8c0aa8e286829d98416b0da3c7d8474fb723cb984522edf69a12", "operation": "MaskedSoftmax", "operators": [ "Div", "Equal", "Exp", "ReduceMax", "ReduceSum", "Sub", "Where" ], "opset": 19, "optimization": "disabled", "output_hash": "32a649a8692e9d97899af94746f67734e71d0f07cb228f4ecb2635ea9a0da2a7", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.008375, "p95_ms": 0.009382999999999999, "repeats": 7, "samples_ms": [ 0.009458, 0.009208, 0.008958, 0.008208, 0.008208, 0.008375, 0.007958 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.053375, "p95_ms": 0.0569374, "repeats": 7, "samples_ms": [ 0.054458, 0.058, 0.051875, 0.053375, 0.050333, 0.053917, 0.051875 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 0.0005, "rtol": 0.005 }, "variant": "ordinary", "warnings": [ { "backend": "reference", "message": "underflow encountered in exp" } ] }, { "artifacts": { "bundle": "observations/07fbcef46e39ad234de58a5eae350cf99b8b30d54101898cafa13dbf76c30263/reproducer.zip", "bundle_sha256": "32f35ff7bd71a3ad1309f900e037dde0b012290085ed1d71eab4063e2240d360", "case": "artifacts/dd4a323c40f3e84c9085f3f16927127d73bf1069e2ef9159d5fff6742ece5ea4/case.json", "inputs": "artifacts/dd4a323c40f3e84c9085f3f16927127d73bf1069e2ef9159d5fff6742ece5ea4/inputs.npz", "manifest": "artifacts/dd4a323c40f3e84c9085f3f16927127d73bf1069e2ef9159d5fff6742ece5ea4/manifest.json", "model": "artifacts/dd4a323c40f3e84c9085f3f16927127d73bf1069e2ef9159d5fff6742ece5ea4/model.onnx", "outputs": "observations/07fbcef46e39ad234de58a5eae350cf99b8b30d54101898cafa13dbf76c30263/outputs.npz" }, "candidate_group": "b6568ee6b53114ba0d16e668bb04b71603b562ed6371b7b4de4b247836662f8a", "case_id": "dd4a323c40f3e84c9085f3f16927127d73bf1069e2ef9159d5fff6742ece5ea4", "category": "masking", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "backend": "reference", "name": "probability_range", "passed": true }, { "backend": "reference", "name": "masked_entries_exact_zero", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 5.960464477539063e-08, "max_budget_ratio": 0.005418604070490056, "max_rel": 5.960464477539063e-08, "max_ulp": 268435456, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 17 ], "backend": "reference", "elements": 136, "expected_dtype": "float32", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "backend": "ort", "name": "probability_range", "passed": true }, { "backend": "ort", "name": "masked_entries_exact_zero", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 1.1362135410308838e-07, "max_budget_ratio": 0.01032921400937167, "max_rel": 1.1362135410308838e-07, "max_ulp": 511705088, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 2.9802322387695312e-08, "max_budget_ratio": 0.0070354572694045145, "max_rel": 1.1635067955196146e-07, "max_ulp": 1, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 17 ], "backend": "ort", "elements": 136, "expected_dtype": "float32", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } } ], "comparison": { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 2.9802322387695312e-08, "max_budget_ratio": 0.007035457764381139, "max_rel": 1.1635069308944367e-07, "max_ulp": 1, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, "comparison_key": "18c1873ff4373d81a5837c17fbe99900a20ab275cf0195880e1add388d4f248b", "contract": "masked_probability", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "mask": { "dtype": "bool", "shape": [ 4, 17 ] }, "x": { "dtype": "float32", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "MaskedSoftmax/ordinary/float32/4x17/seed-20260906", "observation_id": "07fbcef46e39ad234de58a5eae350cf99b8b30d54101898cafa13dbf76c30263", "operation": "MaskedSoftmax", "operators": [ "Div", "Equal", "Exp", "ReduceMax", "ReduceSum", "Sub", "Where" ], "opset": 19, "optimization": "all", "output_hash": "3ecb5895175b4c4dc8e58977237f30612384d985beb1029139c85cf8d333c4c3", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.007416, "p95_ms": 0.008179500000000001, "repeats": 7, "samples_ms": [ 0.007917, 0.008292, 0.007708, 0.007416, 0.007334, 0.00725, 0.007166 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.050833, "p95_ms": 0.07183749999999998, "repeats": 7, "samples_ms": [ 0.05025, 0.048959, 0.051625, 0.048208, 0.050833, 0.062125, 0.076 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-06, "rtol": 1e-05 }, "variant": "ordinary", "warnings": [ { "backend": "reference", "message": "underflow encountered in exp" } ] }, { "artifacts": { "bundle": "observations/ffa56887e404113d639fd4cc0ec9fd2f75fa4bc380fd508bbdfba61bddf9b7a2/reproducer.zip", "bundle_sha256": "eed00520337431fcb6657b842f0a9a3b0a367fd2320c0c1d1d4701c9d9169095", "case": "artifacts/dd4a323c40f3e84c9085f3f16927127d73bf1069e2ef9159d5fff6742ece5ea4/case.json", "inputs": "artifacts/dd4a323c40f3e84c9085f3f16927127d73bf1069e2ef9159d5fff6742ece5ea4/inputs.npz", "manifest": "artifacts/dd4a323c40f3e84c9085f3f16927127d73bf1069e2ef9159d5fff6742ece5ea4/manifest.json", "model": "artifacts/dd4a323c40f3e84c9085f3f16927127d73bf1069e2ef9159d5fff6742ece5ea4/model.onnx", "outputs": "observations/ffa56887e404113d639fd4cc0ec9fd2f75fa4bc380fd508bbdfba61bddf9b7a2/outputs.npz" }, "candidate_group": "b6568ee6b53114ba0d16e668bb04b71603b562ed6371b7b4de4b247836662f8a", "case_id": "dd4a323c40f3e84c9085f3f16927127d73bf1069e2ef9159d5fff6742ece5ea4", "category": "masking", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "backend": "reference", "name": "probability_range", "passed": true }, { "backend": "reference", "name": "masked_entries_exact_zero", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 5.960464477539063e-08, "max_budget_ratio": 0.005418604070490056, "max_rel": 5.960464477539063e-08, "max_ulp": 268435456, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 17 ], "backend": "reference", "elements": 136, "expected_dtype": "float32", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "backend": "ort", "name": "probability_range", "passed": true }, { "backend": "ort", "name": "masked_entries_exact_zero", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 1.1362135410308838e-07, "max_budget_ratio": 0.01032921400937167, "max_rel": 1.1362135410308838e-07, "max_ulp": 511705088, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 2.9802322387695312e-08, "max_budget_ratio": 0.0070354572694045145, "max_rel": 1.1635067955196146e-07, "max_ulp": 1, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 17 ], "backend": "ort", "elements": 136, "expected_dtype": "float32", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } } ], "comparison": { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 2.9802322387695312e-08, "max_budget_ratio": 0.007035457764381139, "max_rel": 1.1635069308944367e-07, "max_ulp": 1, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, "comparison_key": "f585159eff17031af216f8c2e70240c639a58222e07f30705b08e086019efe12", "contract": "masked_probability", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "mask": { "dtype": "bool", "shape": [ 4, 17 ] }, "x": { "dtype": "float32", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "MaskedSoftmax/ordinary/float32/4x17/seed-20260906", "observation_id": "ffa56887e404113d639fd4cc0ec9fd2f75fa4bc380fd508bbdfba61bddf9b7a2", "operation": "MaskedSoftmax", "operators": [ "Div", "Equal", "Exp", "ReduceMax", "ReduceSum", "Sub", "Where" ], "opset": 19, "optimization": "disabled", "output_hash": "3ecb5895175b4c4dc8e58977237f30612384d985beb1029139c85cf8d333c4c3", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.006791, "p95_ms": 0.0071463, "repeats": 7, "samples_ms": [ 0.007209, 0.007, 0.006791, 0.006709, 0.006958, 0.006333, 0.006167 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.049709, "p95_ms": 0.0514043, "repeats": 7, "samples_ms": [ 0.051917, 0.050208, 0.05, 0.049709, 0.048833, 0.048958, 0.049417 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-06, "rtol": 1e-05 }, "variant": "ordinary", "warnings": [ { "backend": "reference", "message": "underflow encountered in exp" } ] }, { "artifacts": { "bundle": "observations/fd88cdd5b273cd30655203e73ff190693ef009fccd2a0ba36a012ea1e006fb9d/reproducer.zip", "bundle_sha256": "4681c4526f920d56620d22a8b5b79fee361ece73bcd091ce8ffac3268e4326c1", "case": "artifacts/6e7a1e43632344c21a1d7036a91db76c987a7d7e1f22e1e1b9ee7d1f4098cc31/case.json", "inputs": "artifacts/6e7a1e43632344c21a1d7036a91db76c987a7d7e1f22e1e1b9ee7d1f4098cc31/inputs.npz", "manifest": "artifacts/6e7a1e43632344c21a1d7036a91db76c987a7d7e1f22e1e1b9ee7d1f4098cc31/manifest.json", "model": "artifacts/6e7a1e43632344c21a1d7036a91db76c987a7d7e1f22e1e1b9ee7d1f4098cc31/model.onnx", "outputs": "observations/fd88cdd5b273cd30655203e73ff190693ef009fccd2a0ba36a012ea1e006fb9d/outputs.npz" }, "candidate_group": "263565172e8cf0c9139bb17ee25c5c77e2ea165c94d9b734a3b0db56775ddd01", "case_id": "6e7a1e43632344c21a1d7036a91db76c987a7d7e1f22e1e1b9ee7d1f4098cc31", "category": "masking", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "backend": "reference", "name": "probability_range", "passed": true }, { "backend": "reference", "name": "masked_entries_exact_zero", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 2.220446049250313e-16, "max_budget_ratio": 0.00020185873175002848, "max_rel": 2.220446049250313e-16, "max_ulp": 2, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 8, 17 ], "backend": "reference", "elements": 136, "expected_dtype": "float64", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "backend": "ort", "name": "probability_range", "passed": true }, { "backend": "ort", "name": "masked_entries_exact_zero", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 2.220446049250313e-16, "max_budget_ratio": 0.00020185873175002848, "max_rel": 2.220446049250313e-16, "max_ulp": 1, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 1.3877787807814457e-17, "max_budget_ratio": 7.979974685408774e-05, "max_rel": 1.8777196411177897e-16, "max_ulp": 1, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 5.551115123125783e-17, "max_budget_ratio": 0.00018005361025368, "max_rel": 2.6649173723998484e-16, "max_ulp": 2, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 8, 17 ], "backend": "ort", "elements": 136, "expected_dtype": "float64", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } } ], "comparison": { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 5.551115123125783e-17, "max_budget_ratio": 0.00018005361025368006, "max_rel": 3.755439282235581e-16, "max_ulp": 2, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, "comparison_key": "2d4f8eb566477cfb2932465776184f74c78d6b65c6be467f7460ab226dd16f39", "contract": "masked_probability", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "mask": { "dtype": "bool", "shape": [ 4, 17 ] }, "x": { "dtype": "float64", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "MaskedSoftmax/ordinary/float64/4x17/seed-20260906", "observation_id": "fd88cdd5b273cd30655203e73ff190693ef009fccd2a0ba36a012ea1e006fb9d", "operation": "MaskedSoftmax", "operators": [ "Div", "Equal", "Exp", "ReduceMax", "ReduceSum", "Sub", "Where" ], "opset": 19, "optimization": "all", "output_hash": "0085c088681693423df9a9d44f00335a17301e161a3c4207a3af3d41b5d944b2", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.007625, "p95_ms": 0.0081999, "repeats": 7, "samples_ms": [ 0.00825, 0.008083, 0.007625, 0.007417, 0.007625, 0.007334, 0.007333 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.048583, "p95_ms": 0.051803699999999994, "repeats": 7, "samples_ms": [ 0.052416, 0.050375, 0.049333, 0.048583, 0.048209, 0.048542, 0.048459 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-13, "rtol": 1e-12 }, "variant": "ordinary", "warnings": [ { "backend": "reference", "message": "underflow encountered in exp" } ] }, { "artifacts": { "bundle": "observations/d70ae1faec2aeb2dc3df46096f16c7525bc281390444c95605681b4aac36c3ca/reproducer.zip", "bundle_sha256": "bb005edb35b4cfc277385c474b940d1a4d20b64b2ac9e3ab8e253744a0acbf1e", "case": "artifacts/6e7a1e43632344c21a1d7036a91db76c987a7d7e1f22e1e1b9ee7d1f4098cc31/case.json", "inputs": "artifacts/6e7a1e43632344c21a1d7036a91db76c987a7d7e1f22e1e1b9ee7d1f4098cc31/inputs.npz", "manifest": "artifacts/6e7a1e43632344c21a1d7036a91db76c987a7d7e1f22e1e1b9ee7d1f4098cc31/manifest.json", "model": "artifacts/6e7a1e43632344c21a1d7036a91db76c987a7d7e1f22e1e1b9ee7d1f4098cc31/model.onnx", "outputs": "observations/d70ae1faec2aeb2dc3df46096f16c7525bc281390444c95605681b4aac36c3ca/outputs.npz" }, "candidate_group": "263565172e8cf0c9139bb17ee25c5c77e2ea165c94d9b734a3b0db56775ddd01", "case_id": "6e7a1e43632344c21a1d7036a91db76c987a7d7e1f22e1e1b9ee7d1f4098cc31", "category": "masking", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "backend": "reference", "name": "probability_range", "passed": true }, { "backend": "reference", "name": "masked_entries_exact_zero", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 2.220446049250313e-16, "max_budget_ratio": 0.00020185873175002848, "max_rel": 2.220446049250313e-16, "max_ulp": 2, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 8, 17 ], "backend": "reference", "elements": 136, "expected_dtype": "float64", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "backend": "ort", "name": "probability_range", "passed": true }, { "backend": "ort", "name": "masked_entries_exact_zero", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 2.220446049250313e-16, "max_budget_ratio": 0.00020185873175002848, "max_rel": 2.220446049250313e-16, "max_ulp": 1, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 1.3877787807814457e-17, "max_budget_ratio": 7.979974685408774e-05, "max_rel": 1.8777196411177897e-16, "max_ulp": 1, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 5.551115123125783e-17, "max_budget_ratio": 0.00018005361025368, "max_rel": 2.6649173723998484e-16, "max_ulp": 2, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 8, 17 ], "backend": "ort", "elements": 136, "expected_dtype": "float64", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } } ], "comparison": { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 5.551115123125783e-17, "max_budget_ratio": 0.00018005361025368006, "max_rel": 3.755439282235581e-16, "max_ulp": 2, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, "comparison_key": "bb8ad5cf745adf3a07d6080fa0f91206b9df7d19df570f42500f9be0a952907c", "contract": "masked_probability", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "mask": { "dtype": "bool", "shape": [ 4, 17 ] }, "x": { "dtype": "float64", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "MaskedSoftmax/ordinary/float64/4x17/seed-20260906", "observation_id": "d70ae1faec2aeb2dc3df46096f16c7525bc281390444c95605681b4aac36c3ca", "operation": "MaskedSoftmax", "operators": [ "Div", "Equal", "Exp", "ReduceMax", "ReduceSum", "Sub", "Where" ], "opset": 19, "optimization": "disabled", "output_hash": "0085c088681693423df9a9d44f00335a17301e161a3c4207a3af3d41b5d944b2", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.007416, "p95_ms": 0.0078252, "repeats": 7, "samples_ms": [ 0.007709, 0.007875, 0.007416, 0.007208, 0.007459, 0.006458, 0.006209 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.04975, "p95_ms": 0.0513207, "repeats": 7, "samples_ms": [ 0.051708, 0.050417, 0.050125, 0.049584, 0.049, 0.04975, 0.044666 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-13, "rtol": 1e-12 }, "variant": "ordinary", "warnings": [ { "backend": "reference", "message": "underflow encountered in exp" } ] }, { "artifacts": { "bundle": "observations/2f6847726fe38bb33778a6930a74fa269b6af453bc32e80684ee2bc1b17bedde/reproducer.zip", "bundle_sha256": "036ca1f52fdc25ebfc2b91a119310b469ba110c4252e912f5faa74d1d8306238", "case": "artifacts/6f8806e32546c96d2fbaf6ee6534220a69ab3e313dfd6c777530028e43848121/case.json", "inputs": "artifacts/6f8806e32546c96d2fbaf6ee6534220a69ab3e313dfd6c777530028e43848121/inputs.npz", "manifest": "artifacts/6f8806e32546c96d2fbaf6ee6534220a69ab3e313dfd6c777530028e43848121/manifest.json", "model": "artifacts/6f8806e32546c96d2fbaf6ee6534220a69ab3e313dfd6c777530028e43848121/model.onnx", "outputs": "observations/2f6847726fe38bb33778a6930a74fa269b6af453bc32e80684ee2bc1b17bedde/outputs.npz" }, "candidate_group": "baa15bc55c0b9cbe09a19028ad22e69b778b47734e3c79cab09bc42d2cd5090f", "case_id": "6f8806e32546c96d2fbaf6ee6534220a69ab3e313dfd6c777530028e43848121", "category": "masking", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "backend": "reference", "name": "probability_range", "passed": true }, { "backend": "reference", "name": "masked_entries_exact_zero", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 7.264316082000732e-08, "max_budget_ratio": 0.006603923710909756, "max_rel": 7.264316082000732e-08, "max_ulp": 654311424, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 120 ], "backend": "reference", "elements": 480, "expected_dtype": "float32", "expected_shape": [ 4, 120 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 120 ], "backend": "reference", "elements": 480, "expected_dtype": "float32", "expected_shape": [ 4, 120 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 120 ], "backend": "reference", "elements": 480, "expected_dtype": "float32", "expected_shape": [ 4, 120 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 120 ], "backend": "reference", "elements": 960, "expected_dtype": "float32", "expected_shape": [ 8, 120 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "backend": "ort", "name": "probability_range", "passed": true }, { "backend": "ort", "name": "masked_entries_exact_zero", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 5.774199962615967e-08, "max_budget_ratio": 0.005249272693287242, "max_rel": 5.774199962615967e-08, "max_ulp": 520093696, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 120 ], "backend": "ort", "elements": 480, "expected_dtype": "float32", "expected_shape": [ 4, 120 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 120 ], "backend": "ort", "elements": 480, "expected_dtype": "float32", "expected_shape": [ 4, 120 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 120 ], "backend": "ort", "elements": 480, "expected_dtype": "float32", "expected_shape": [ 4, 120 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 120 ], "backend": "ort", "elements": 960, "expected_dtype": "float32", "expected_shape": [ 8, 120 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } } ], "comparison": { "actual_dtype": "float32", "actual_shape": [ 4, 120 ], "elements": 480, "expected_dtype": "float32", "expected_shape": [ 4, 120 ], "first_mismatch": null, "max_abs": 1.862645149230957e-09, "max_budget_ratio": 0.0016021754415049174, "max_rel": 1.1457318167204893e-07, "max_ulp": 1, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, "comparison_key": "a83e413b95436f901cb5b9e36b4d5cd4538a5c744a17c8c30b3470eae60a935f", "contract": "masked_probability", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "mask": { "dtype": "bool", "shape": [ 4, 120 ] }, "x": { "dtype": "float32", "shape": [ 4, 120 ] } }, "ir_version": 10, "name": "MaskedSoftmax/random-002/float32/4x120/seed-20260906", "observation_id": "2f6847726fe38bb33778a6930a74fa269b6af453bc32e80684ee2bc1b17bedde", "operation": "MaskedSoftmax", "operators": [ "Div", "Equal", "Exp", "ReduceMax", "ReduceSum", "Sub", "Where" ], "opset": 19, "optimization": "all", "output_hash": "1454fe2856a05da4132d13e91df1a5a753d6f7d6c1c8621417d8062309d99571", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.008125, "p95_ms": 0.0087208, "repeats": 7, "samples_ms": [ 0.008833, 0.008459, 0.007959, 0.008208, 0.008125, 0.007542, 0.007375 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.052334, "p95_ms": 0.0560417, "repeats": 7, "samples_ms": [ 0.053292, 0.052334, 0.0515, 0.050459, 0.051, 0.056417, 0.055166 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-06, "rtol": 1e-05 }, "variant": "random-002", "warnings": [ { "backend": "reference", "message": "underflow encountered in exp" } ] }, { "artifacts": { "bundle": "observations/c611933192d77d493d42e8089f8becdd945d66f9c893f0f7b040dafba2cfe369/reproducer.zip", "bundle_sha256": "eb625755ceedc968f2e107b907d35952a3d20d6cb5d77ae1f336dea6b8dc2303", "case": "artifacts/6f8806e32546c96d2fbaf6ee6534220a69ab3e313dfd6c777530028e43848121/case.json", "inputs": "artifacts/6f8806e32546c96d2fbaf6ee6534220a69ab3e313dfd6c777530028e43848121/inputs.npz", "manifest": "artifacts/6f8806e32546c96d2fbaf6ee6534220a69ab3e313dfd6c777530028e43848121/manifest.json", "model": "artifacts/6f8806e32546c96d2fbaf6ee6534220a69ab3e313dfd6c777530028e43848121/model.onnx", "outputs": "observations/c611933192d77d493d42e8089f8becdd945d66f9c893f0f7b040dafba2cfe369/outputs.npz" }, "candidate_group": "baa15bc55c0b9cbe09a19028ad22e69b778b47734e3c79cab09bc42d2cd5090f", "case_id": "6f8806e32546c96d2fbaf6ee6534220a69ab3e313dfd6c777530028e43848121", "category": "masking", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "backend": "reference", "name": "probability_range", "passed": true }, { "backend": "reference", "name": "masked_entries_exact_zero", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 7.264316082000732e-08, "max_budget_ratio": 0.006603923710909756, "max_rel": 7.264316082000732e-08, "max_ulp": 654311424, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 120 ], "backend": "reference", "elements": 480, "expected_dtype": "float32", "expected_shape": [ 4, 120 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 120 ], "backend": "reference", "elements": 480, "expected_dtype": "float32", "expected_shape": [ 4, 120 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 120 ], "backend": "reference", "elements": 480, "expected_dtype": "float32", "expected_shape": [ 4, 120 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 120 ], "backend": "reference", "elements": 960, "expected_dtype": "float32", "expected_shape": [ 8, 120 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "backend": "ort", "name": "probability_range", "passed": true }, { "backend": "ort", "name": "masked_entries_exact_zero", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 5.774199962615967e-08, "max_budget_ratio": 0.005249272693287242, "max_rel": 5.774199962615967e-08, "max_ulp": 520093696, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 120 ], "backend": "ort", "elements": 480, "expected_dtype": "float32", "expected_shape": [ 4, 120 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 120 ], "backend": "ort", "elements": 480, "expected_dtype": "float32", "expected_shape": [ 4, 120 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 120 ], "backend": "ort", "elements": 480, "expected_dtype": "float32", "expected_shape": [ 4, 120 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 120 ], "backend": "ort", "elements": 960, "expected_dtype": "float32", "expected_shape": [ 8, 120 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } } ], "comparison": { "actual_dtype": "float32", "actual_shape": [ 4, 120 ], "elements": 480, "expected_dtype": "float32", "expected_shape": [ 4, 120 ], "first_mismatch": null, "max_abs": 1.862645149230957e-09, "max_budget_ratio": 0.0016021754415049174, "max_rel": 1.1457318167204893e-07, "max_ulp": 1, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, "comparison_key": "ec9be0a127343dcae3b919b9649ff094e9c2fbb539facb328c8f310dca0b7ed0", "contract": "masked_probability", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "mask": { "dtype": "bool", "shape": [ 4, 120 ] }, "x": { "dtype": "float32", "shape": [ 4, 120 ] } }, "ir_version": 10, "name": "MaskedSoftmax/random-002/float32/4x120/seed-20260906", "observation_id": "c611933192d77d493d42e8089f8becdd945d66f9c893f0f7b040dafba2cfe369", "operation": "MaskedSoftmax", "operators": [ "Div", "Equal", "Exp", "ReduceMax", "ReduceSum", "Sub", "Where" ], "opset": 19, "optimization": "disabled", "output_hash": "1454fe2856a05da4132d13e91df1a5a753d6f7d6c1c8621417d8062309d99571", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.00775, "p95_ms": 0.0083123, "repeats": 7, "samples_ms": [ 0.008166, 0.008375, 0.00775, 0.007917, 0.007667, 0.007584, 0.007333 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.051208, "p95_ms": 0.0545627, "repeats": 7, "samples_ms": [ 0.05525, 0.052959, 0.051875, 0.051084, 0.050667, 0.051, 0.051208 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-06, "rtol": 1e-05 }, "variant": "random-002", "warnings": [ { "backend": "reference", "message": "underflow encountered in exp" } ] }, { "artifacts": { "bundle": "observations/1959361dd3dba3de3c5a6b5abe4f702d857648c234f6bd8748b0a08da315b07c/reproducer.zip", "bundle_sha256": "375ab2590c93d5ef352fa74394a711a34caa9392e7772c55635c699cae097074", "case": "artifacts/0ed85eaf53bbafee46e4dd5660feb8995acdd33f96eec60cea1bd31407aeacba/case.json", "inputs": "artifacts/0ed85eaf53bbafee46e4dd5660feb8995acdd33f96eec60cea1bd31407aeacba/inputs.npz", "manifest": "artifacts/0ed85eaf53bbafee46e4dd5660feb8995acdd33f96eec60cea1bd31407aeacba/manifest.json", "model": "artifacts/0ed85eaf53bbafee46e4dd5660feb8995acdd33f96eec60cea1bd31407aeacba/model.onnx", "outputs": "observations/1959361dd3dba3de3c5a6b5abe4f702d857648c234f6bd8748b0a08da315b07c/outputs.npz" }, "candidate_group": "985788400de345f439575e8c96d141d063ef68cfe3dbf4147fa1cdcf082f3ae7", "case_id": "0ed85eaf53bbafee46e4dd5660feb8995acdd33f96eec60cea1bd31407aeacba", "category": "masking", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "backend": "reference", "name": "probability_range", "passed": true }, { "backend": "reference", "name": "masked_entries_exact_zero", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 4.2970723646007514e-08, "max_budget_ratio": 0.003906429422364319, "max_rel": 4.2970723646007514e-08, "max_ulp": 234880753, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 117 ], "backend": "reference", "elements": 468, "expected_dtype": "float32", "expected_shape": [ 4, 117 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 117 ], "backend": "reference", "elements": 468, "expected_dtype": "float32", "expected_shape": [ 4, 117 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 117 ], "backend": "reference", "elements": 468, "expected_dtype": "float32", "expected_shape": [ 4, 117 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 117 ], "backend": "reference", "elements": 936, "expected_dtype": "float32", "expected_shape": [ 8, 117 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "backend": "ort", "name": "probability_range", "passed": true }, { "backend": "ort", "name": "masked_entries_exact_zero", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 2.7939647151420388e-08, "max_budget_ratio": 0.0025399679228563987, "max_rel": 2.7939647151420388e-08, "max_ulp": 251657969, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 117 ], "backend": "ort", "elements": 468, "expected_dtype": "float32", "expected_shape": [ 4, 117 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 117 ], "backend": "ort", "elements": 468, "expected_dtype": "float32", "expected_shape": [ 4, 117 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 117 ], "backend": "ort", "elements": 468, "expected_dtype": "float32", "expected_shape": [ 4, 117 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 117 ], "backend": "ort", "elements": 936, "expected_dtype": "float32", "expected_shape": [ 8, 117 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } } ], "comparison": { "actual_dtype": "float32", "actual_shape": [ 4, 117 ], "elements": 468, "expected_dtype": "float32", "expected_shape": [ 4, 117 ], "first_mismatch": null, "max_abs": 2.9802322387695312e-08, "max_budget_ratio": 0.005747742105846948, "max_rel": 1.6057894490480198e-07, "max_ulp": 2, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, "comparison_key": "403ddff24b9a02ca7fbf64b8881502a4dcc14e57e46a59c628cb3e0d73cdead3", "contract": "masked_probability", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "mask": { "dtype": "bool", "shape": [ 4, 117 ] }, "x": { "dtype": "float32", "shape": [ 4, 117 ] } }, "ir_version": 10, "name": "MaskedSoftmax/random-009/float32/4x117/seed-20260906", "observation_id": "1959361dd3dba3de3c5a6b5abe4f702d857648c234f6bd8748b0a08da315b07c", "operation": "MaskedSoftmax", "operators": [ "Div", "Equal", "Exp", "ReduceMax", "ReduceSum", "Sub", "Where" ], "opset": 19, "optimization": "all", "output_hash": "3fcc7ea4a781897e33960eb450b430fb264a50272148360044b7b4e6f9f9bd98", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.007875, "p95_ms": 0.012554399999999995, "repeats": 7, "samples_ms": [ 0.008375, 0.0085, 0.007875, 0.007875, 0.007875, 0.007417, 0.014292 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.051292, "p95_ms": 0.0546787, "repeats": 7, "samples_ms": [ 0.055666, 0.052375, 0.051417, 0.051208, 0.051292, 0.051208, 0.051042 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-06, "rtol": 1e-05 }, "variant": "random-009", "warnings": [ { "backend": "reference", "message": "underflow encountered in divide" }, { "backend": "reference", "message": "underflow encountered in exp" } ] }, { "artifacts": { "bundle": "observations/6aea086d027a33d5b1840f820e98a893b9150c41cf5eb934a883415aec7e15f6/reproducer.zip", "bundle_sha256": "4bef70624201393b9506158cefbce050eb52cb7274d72ab2b0274cff13c666a2", "case": "artifacts/0ed85eaf53bbafee46e4dd5660feb8995acdd33f96eec60cea1bd31407aeacba/case.json", "inputs": "artifacts/0ed85eaf53bbafee46e4dd5660feb8995acdd33f96eec60cea1bd31407aeacba/inputs.npz", "manifest": "artifacts/0ed85eaf53bbafee46e4dd5660feb8995acdd33f96eec60cea1bd31407aeacba/manifest.json", "model": "artifacts/0ed85eaf53bbafee46e4dd5660feb8995acdd33f96eec60cea1bd31407aeacba/model.onnx", "outputs": "observations/6aea086d027a33d5b1840f820e98a893b9150c41cf5eb934a883415aec7e15f6/outputs.npz" }, "candidate_group": "985788400de345f439575e8c96d141d063ef68cfe3dbf4147fa1cdcf082f3ae7", "case_id": "0ed85eaf53bbafee46e4dd5660feb8995acdd33f96eec60cea1bd31407aeacba", "category": "masking", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "backend": "reference", "name": "probability_range", "passed": true }, { "backend": "reference", "name": "masked_entries_exact_zero", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 4.2970723646007514e-08, "max_budget_ratio": 0.003906429422364319, "max_rel": 4.2970723646007514e-08, "max_ulp": 234880753, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 117 ], "backend": "reference", "elements": 468, "expected_dtype": "float32", "expected_shape": [ 4, 117 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 117 ], "backend": "reference", "elements": 468, "expected_dtype": "float32", "expected_shape": [ 4, 117 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 117 ], "backend": "reference", "elements": 468, "expected_dtype": "float32", "expected_shape": [ 4, 117 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 117 ], "backend": "reference", "elements": 936, "expected_dtype": "float32", "expected_shape": [ 8, 117 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "backend": "ort", "name": "probability_range", "passed": true }, { "backend": "ort", "name": "masked_entries_exact_zero", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 2.7939647151420388e-08, "max_budget_ratio": 0.0025399679228563987, "max_rel": 2.7939647151420388e-08, "max_ulp": 251657969, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 117 ], "backend": "ort", "elements": 468, "expected_dtype": "float32", "expected_shape": [ 4, 117 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 117 ], "backend": "ort", "elements": 468, "expected_dtype": "float32", "expected_shape": [ 4, 117 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 117 ], "backend": "ort", "elements": 468, "expected_dtype": "float32", "expected_shape": [ 4, 117 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 117 ], "backend": "ort", "elements": 936, "expected_dtype": "float32", "expected_shape": [ 8, 117 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } } ], "comparison": { "actual_dtype": "float32", "actual_shape": [ 4, 117 ], "elements": 468, "expected_dtype": "float32", "expected_shape": [ 4, 117 ], "first_mismatch": null, "max_abs": 2.9802322387695312e-08, "max_budget_ratio": 0.005747742105846948, "max_rel": 1.6057894490480198e-07, "max_ulp": 2, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, "comparison_key": "0ce30345533d0f52f353b973c923216a4f4f861cf7d9ec04927998eed9dbf752", "contract": "masked_probability", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "mask": { "dtype": "bool", "shape": [ 4, 117 ] }, "x": { "dtype": "float32", "shape": [ 4, 117 ] } }, "ir_version": 10, "name": "MaskedSoftmax/random-009/float32/4x117/seed-20260906", "observation_id": "6aea086d027a33d5b1840f820e98a893b9150c41cf5eb934a883415aec7e15f6", "operation": "MaskedSoftmax", "operators": [ "Div", "Equal", "Exp", "ReduceMax", "ReduceSum", "Sub", "Where" ], "opset": 19, "optimization": "disabled", "output_hash": "3fcc7ea4a781897e33960eb450b430fb264a50272148360044b7b4e6f9f9bd98", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.007708, "p95_ms": 0.0084874, "repeats": 7, "samples_ms": [ 0.0085, 0.008458, 0.00775, 0.007667, 0.007708, 0.0075, 0.00725 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.051833, "p95_ms": 0.0543749, "repeats": 7, "samples_ms": [ 0.05525, 0.052333, 0.051833, 0.051416, 0.051042, 0.051875, 0.050833 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-06, "rtol": 1e-05 }, "variant": "random-009", "warnings": [ { "backend": "reference", "message": "underflow encountered in divide" }, { "backend": "reference", "message": "underflow encountered in exp" } ] }, { "artifacts": { "bundle": "observations/44c70629b770f37d255c4bfd4fb760c9ef66c224c5c36b00003dd9979b8d8c42/reproducer.zip", "bundle_sha256": "346a1950ec66b1ffbe77f34e52bc49c61d0c46645d9d9581c7b0e8d1a3916b5d", "case": "artifacts/fb11854f14f3a2fcf982e2002d83204e566d2f6c6ec62714061b0d9a9310e378/case.json", "inputs": "artifacts/fb11854f14f3a2fcf982e2002d83204e566d2f6c6ec62714061b0d9a9310e378/inputs.npz", "manifest": "artifacts/fb11854f14f3a2fcf982e2002d83204e566d2f6c6ec62714061b0d9a9310e378/manifest.json", "model": "artifacts/fb11854f14f3a2fcf982e2002d83204e566d2f6c6ec62714061b0d9a9310e378/model.onnx", "outputs": "observations/44c70629b770f37d255c4bfd4fb760c9ef66c224c5c36b00003dd9979b8d8c42/outputs.npz" }, "candidate_group": "1a2ee1152d3d3096f923776db21de838f3fb28f6736c4c5ed10a52ffddf6c090", "case_id": "fb11854f14f3a2fcf982e2002d83204e566d2f6c6ec62714061b0d9a9310e378", "category": "masking", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "backend": "reference", "name": "probability_range", "passed": true }, { "backend": "reference", "name": "masked_entries_exact_zero", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 9.598659289622447e-08, "max_budget_ratio": 0.008726053899656768, "max_rel": 9.598659289622447e-08, "max_ulp": 864570368, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 84 ], "backend": "reference", "elements": 336, "expected_dtype": "float32", "expected_shape": [ 4, 84 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 84 ], "backend": "reference", "elements": 336, "expected_dtype": "float32", "expected_shape": [ 4, 84 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 84 ], "backend": "reference", "elements": 336, "expected_dtype": "float32", "expected_shape": [ 4, 84 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 84 ], "backend": "reference", "elements": 672, "expected_dtype": "float32", "expected_shape": [ 8, 84 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "backend": "ort", "name": "probability_range", "passed": true }, { "backend": "ort", "name": "masked_entries_exact_zero", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 2.3371057977783494e-08, "max_budget_ratio": 0.0021246416343439537, "max_rel": 2.3371057977783494e-08, "max_ulp": 105253888, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 84 ], "backend": "ort", "elements": 336, "expected_dtype": "float32", "expected_shape": [ 4, 84 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 84 ], "backend": "ort", "elements": 336, "expected_dtype": "float32", "expected_shape": [ 4, 84 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 84 ], "backend": "ort", "elements": 336, "expected_dtype": "float32", "expected_shape": [ 4, 84 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 84 ], "backend": "ort", "elements": 672, "expected_dtype": "float32", "expected_shape": [ 8, 84 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } } ], "comparison": { "actual_dtype": "float32", "actual_shape": [ 4, 84 ], "elements": 336, "expected_dtype": "float32", "expected_shape": [ 4, 84 ], "first_mismatch": null, "max_abs": 1.1920928955078125e-07, "max_budget_ratio": 0.010856747410975158, "max_rel": 2.1825233352666852e-07, "max_ulp": 2, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, "comparison_key": "92358540c9edfc904c2250cc182f808da5af860b73085b5db3aa68bd3d92de43", "contract": "masked_probability", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "mask": { "dtype": "bool", "shape": [ 4, 84 ] }, "x": { "dtype": "float32", "shape": [ 4, 84 ] } }, "ir_version": 10, "name": "MaskedSoftmax/random-016/float32/4x84/seed-20260906", "observation_id": "44c70629b770f37d255c4bfd4fb760c9ef66c224c5c36b00003dd9979b8d8c42", "operation": "MaskedSoftmax", "operators": [ "Div", "Equal", "Exp", "ReduceMax", "ReduceSum", "Sub", "Where" ], "opset": 19, "optimization": "all", "output_hash": "71018e5424f75adf1f2f3535aff77f306f76ee5d8bbd815047f9d2b1a80d4daa", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.007791, "p95_ms": 0.0086418, "repeats": 7, "samples_ms": [ 0.008667, 0.008583, 0.007917, 0.00775, 0.007791, 0.007333, 0.007167 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.049542, "p95_ms": 0.050862599999999994, "repeats": 7, "samples_ms": [ 0.051, 0.05, 0.049542, 0.048458, 0.049417, 0.050542, 0.049292 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-06, "rtol": 1e-05 }, "variant": "random-016", "warnings": [ { "backend": "reference", "message": "underflow encountered in exp" } ] }, { "artifacts": { "bundle": "observations/e3bc718b57ecd57e42efc31ec9c32147364d84cebcd9c419efbefab6f073e973/reproducer.zip", "bundle_sha256": "9a4f3f444497c54cee60806071f06fd1596c9039b45e814928dafd925a17fc5c", "case": "artifacts/fb11854f14f3a2fcf982e2002d83204e566d2f6c6ec62714061b0d9a9310e378/case.json", "inputs": "artifacts/fb11854f14f3a2fcf982e2002d83204e566d2f6c6ec62714061b0d9a9310e378/inputs.npz", "manifest": "artifacts/fb11854f14f3a2fcf982e2002d83204e566d2f6c6ec62714061b0d9a9310e378/manifest.json", "model": "artifacts/fb11854f14f3a2fcf982e2002d83204e566d2f6c6ec62714061b0d9a9310e378/model.onnx", "outputs": "observations/e3bc718b57ecd57e42efc31ec9c32147364d84cebcd9c419efbefab6f073e973/outputs.npz" }, "candidate_group": "1a2ee1152d3d3096f923776db21de838f3fb28f6736c4c5ed10a52ffddf6c090", "case_id": "fb11854f14f3a2fcf982e2002d83204e566d2f6c6ec62714061b0d9a9310e378", "category": "masking", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "backend": "reference", "name": "probability_range", "passed": true }, { "backend": "reference", "name": "masked_entries_exact_zero", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 9.598659289622447e-08, "max_budget_ratio": 0.008726053899656768, "max_rel": 9.598659289622447e-08, "max_ulp": 864570368, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 84 ], "backend": "reference", "elements": 336, "expected_dtype": "float32", "expected_shape": [ 4, 84 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 84 ], "backend": "reference", "elements": 336, "expected_dtype": "float32", "expected_shape": [ 4, 84 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 84 ], "backend": "reference", "elements": 336, "expected_dtype": "float32", "expected_shape": [ 4, 84 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 84 ], "backend": "reference", "elements": 672, "expected_dtype": "float32", "expected_shape": [ 8, 84 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "backend": "ort", "name": "probability_range", "passed": true }, { "backend": "ort", "name": "masked_entries_exact_zero", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 2.3371057977783494e-08, "max_budget_ratio": 0.0021246416343439537, "max_rel": 2.3371057977783494e-08, "max_ulp": 105253888, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 84 ], "backend": "ort", "elements": 336, "expected_dtype": "float32", "expected_shape": [ 4, 84 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 84 ], "backend": "ort", "elements": 336, "expected_dtype": "float32", "expected_shape": [ 4, 84 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 84 ], "backend": "ort", "elements": 336, "expected_dtype": "float32", "expected_shape": [ 4, 84 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 84 ], "backend": "ort", "elements": 672, "expected_dtype": "float32", "expected_shape": [ 8, 84 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } } ], "comparison": { "actual_dtype": "float32", "actual_shape": [ 4, 84 ], "elements": 336, "expected_dtype": "float32", "expected_shape": [ 4, 84 ], "first_mismatch": null, "max_abs": 1.1920928955078125e-07, "max_budget_ratio": 0.010856747410975158, "max_rel": 2.1825233352666852e-07, "max_ulp": 2, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, "comparison_key": "8bc2ace5761ddf19d4fa2ec84956bb5f040d580e70140b3bdf0bb418bc07e59e", "contract": "masked_probability", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "mask": { "dtype": "bool", "shape": [ 4, 84 ] }, "x": { "dtype": "float32", "shape": [ 4, 84 ] } }, "ir_version": 10, "name": "MaskedSoftmax/random-016/float32/4x84/seed-20260906", "observation_id": "e3bc718b57ecd57e42efc31ec9c32147364d84cebcd9c419efbefab6f073e973", "operation": "MaskedSoftmax", "operators": [ "Div", "Equal", "Exp", "ReduceMax", "ReduceSum", "Sub", "Where" ], "opset": 19, "optimization": "disabled", "output_hash": "71018e5424f75adf1f2f3535aff77f306f76ee5d8bbd815047f9d2b1a80d4daa", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.007458, "p95_ms": 0.0082751, "repeats": 7, "samples_ms": [ 0.008375, 0.008042, 0.007625, 0.007458, 0.007333, 0.007209, 0.007125 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.04925, "p95_ms": 0.050658100000000005, "repeats": 7, "samples_ms": [ 0.050833, 0.05025, 0.049959, 0.04925, 0.048875, 0.048375, 0.047917 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-06, "rtol": 1e-05 }, "variant": "random-016", "warnings": [ { "backend": "reference", "message": "underflow encountered in exp" } ] }, { "artifacts": { "bundle": "observations/70221e6a79fa697cda33cadb6f72e29e599b80ee794b4f698eab871ea0c3d8de/reproducer.zip", "bundle_sha256": "52988052d1eda3ea83f6aacf13fd3fbc590cf5080f62d0966994f650bfb7430e", "case": "artifacts/54b87cc0e965d6e0cf5649919afec633a75e7f9469f0062c7372eee1d948adf0/case.json", "inputs": "artifacts/54b87cc0e965d6e0cf5649919afec633a75e7f9469f0062c7372eee1d948adf0/inputs.npz", "manifest": "artifacts/54b87cc0e965d6e0cf5649919afec633a75e7f9469f0062c7372eee1d948adf0/manifest.json", "model": "artifacts/54b87cc0e965d6e0cf5649919afec633a75e7f9469f0062c7372eee1d948adf0/model.onnx", "outputs": "observations/70221e6a79fa697cda33cadb6f72e29e599b80ee794b4f698eab871ea0c3d8de/outputs.npz" }, "candidate_group": "a93c6868cc2c919c146528e443d355cc133883ed54b9612a736fd73cc3451ab2", "case_id": "54b87cc0e965d6e0cf5649919afec633a75e7f9469f0062c7372eee1d948adf0", "category": "masking", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "backend": "reference", "name": "probability_range", "passed": true }, { "backend": "reference", "name": "masked_entries_exact_zero", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 6.891787052154541e-08, "max_budget_ratio": 0.0062652609565041276, "max_rel": 6.891787052154541e-08, "max_ulp": 310378496, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 71 ], "backend": "reference", "elements": 284, "expected_dtype": "float32", "expected_shape": [ 4, 71 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 71 ], "backend": "reference", "elements": 284, "expected_dtype": "float32", "expected_shape": [ 4, 71 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 71 ], "backend": "reference", "elements": 284, "expected_dtype": "float32", "expected_shape": [ 4, 71 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 71 ], "backend": "reference", "elements": 568, "expected_dtype": "float32", "expected_shape": [ 8, 71 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "backend": "ort", "name": "probability_range", "passed": true }, { "backend": "ort", "name": "masked_entries_exact_zero", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 6.891787052154541e-08, "max_budget_ratio": 0.0062652609565041276, "max_rel": 6.891787052154541e-08, "max_ulp": 310378496, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 71 ], "backend": "ort", "elements": 284, "expected_dtype": "float32", "expected_shape": [ 4, 71 ], "first_mismatch": null, "max_abs": 3.725290298461914e-09, "max_budget_ratio": 0.0028378931361030904, "max_rel": 1.191346582616371e-07, "max_ulp": 1, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 71 ], "backend": "ort", "elements": 284, "expected_dtype": "float32", "expected_shape": [ 4, 71 ], "first_mismatch": null, "max_abs": 3.725290298461914e-09, "max_budget_ratio": 0.0029221081592562155, "max_rel": 1.3553216195144628e-07, "max_ulp": 2, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 71 ], "backend": "ort", "elements": 284, "expected_dtype": "float32", "expected_shape": [ 4, 71 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 71 ], "backend": "ort", "elements": 568, "expected_dtype": "float32", "expected_shape": [ 8, 71 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } } ], "comparison": { "actual_dtype": "float32", "actual_shape": [ 4, 71 ], "elements": 284, "expected_dtype": "float32", "expected_shape": [ 4, 71 ], "first_mismatch": null, "max_abs": 3.725290298461914e-09, "max_budget_ratio": 0.002969605058800615, "max_rel": 1.4639217937843637e-07, "max_ulp": 2, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, "comparison_key": "22b39efa7f2ac31f7f012c03d0134642a9ffc97f66bf1e0eb6487a70234c1154", "contract": "masked_probability", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "mask": { "dtype": "bool", "shape": [ 4, 71 ] }, "x": { "dtype": "float32", "shape": [ 4, 71 ] } }, "ir_version": 10, "name": "MaskedSoftmax/random-023/float32/4x71/seed-20260906", "observation_id": "70221e6a79fa697cda33cadb6f72e29e599b80ee794b4f698eab871ea0c3d8de", "operation": "MaskedSoftmax", "operators": [ "Div", "Equal", "Exp", "ReduceMax", "ReduceSum", "Sub", "Where" ], "opset": 19, "optimization": "all", "output_hash": "3ac3cb5e2093db1cd1239117bbc9129a8e07e6de4d059a2a34729f51df9d30c0", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.007458, "p95_ms": 0.0079205, "repeats": 7, "samples_ms": [ 0.007958, 0.007833, 0.0075, 0.00725, 0.007458, 0.00725, 0.007084 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.048333, "p95_ms": 0.0501875, "repeats": 7, "samples_ms": [ 0.05075, 0.048875, 0.048333, 0.047959, 0.048583, 0.044833, 0.045667 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-06, "rtol": 1e-05 }, "variant": "random-023", "warnings": [ { "backend": "reference", "message": "underflow encountered in exp" } ] }, { "artifacts": { "bundle": "observations/d012616e76134ed178f50ac43e7ee297ade94bc35b35d09c8c0e54622cff2f4a/reproducer.zip", "bundle_sha256": "ddcaa1bcf77775a524b637fd0777931a2bd6728ceb9f36526ec7094620412744", "case": "artifacts/54b87cc0e965d6e0cf5649919afec633a75e7f9469f0062c7372eee1d948adf0/case.json", "inputs": "artifacts/54b87cc0e965d6e0cf5649919afec633a75e7f9469f0062c7372eee1d948adf0/inputs.npz", "manifest": "artifacts/54b87cc0e965d6e0cf5649919afec633a75e7f9469f0062c7372eee1d948adf0/manifest.json", "model": "artifacts/54b87cc0e965d6e0cf5649919afec633a75e7f9469f0062c7372eee1d948adf0/model.onnx", "outputs": "observations/d012616e76134ed178f50ac43e7ee297ade94bc35b35d09c8c0e54622cff2f4a/outputs.npz" }, "candidate_group": "a93c6868cc2c919c146528e443d355cc133883ed54b9612a736fd73cc3451ab2", "case_id": "54b87cc0e965d6e0cf5649919afec633a75e7f9469f0062c7372eee1d948adf0", "category": "masking", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "backend": "reference", "name": "probability_range", "passed": true }, { "backend": "reference", "name": "masked_entries_exact_zero", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 6.891787052154541e-08, "max_budget_ratio": 0.0062652609565041276, "max_rel": 6.891787052154541e-08, "max_ulp": 310378496, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 71 ], "backend": "reference", "elements": 284, "expected_dtype": "float32", "expected_shape": [ 4, 71 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 71 ], "backend": "reference", "elements": 284, "expected_dtype": "float32", "expected_shape": [ 4, 71 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 71 ], "backend": "reference", "elements": 284, "expected_dtype": "float32", "expected_shape": [ 4, 71 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 71 ], "backend": "reference", "elements": 568, "expected_dtype": "float32", "expected_shape": [ 8, 71 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "backend": "ort", "name": "probability_range", "passed": true }, { "backend": "ort", "name": "masked_entries_exact_zero", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 6.891787052154541e-08, "max_budget_ratio": 0.0062652609565041276, "max_rel": 6.891787052154541e-08, "max_ulp": 310378496, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 71 ], "backend": "ort", "elements": 284, "expected_dtype": "float32", "expected_shape": [ 4, 71 ], "first_mismatch": null, "max_abs": 3.725290298461914e-09, "max_budget_ratio": 0.0028378931361030904, "max_rel": 1.191346582616371e-07, "max_ulp": 1, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 71 ], "backend": "ort", "elements": 284, "expected_dtype": "float32", "expected_shape": [ 4, 71 ], "first_mismatch": null, "max_abs": 3.725290298461914e-09, "max_budget_ratio": 0.0029221081592562155, "max_rel": 1.3553216195144628e-07, "max_ulp": 2, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 71 ], "backend": "ort", "elements": 284, "expected_dtype": "float32", "expected_shape": [ 4, 71 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 71 ], "backend": "ort", "elements": 568, "expected_dtype": "float32", "expected_shape": [ 8, 71 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } } ], "comparison": { "actual_dtype": "float32", "actual_shape": [ 4, 71 ], "elements": 284, "expected_dtype": "float32", "expected_shape": [ 4, 71 ], "first_mismatch": null, "max_abs": 3.725290298461914e-09, "max_budget_ratio": 0.002969605058800615, "max_rel": 1.4639217937843637e-07, "max_ulp": 2, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, "comparison_key": "7c0f103547436598257c111de832452b760dc05a4a7545b2557a382528aaca00", "contract": "masked_probability", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "mask": { "dtype": "bool", "shape": [ 4, 71 ] }, "x": { "dtype": "float32", "shape": [ 4, 71 ] } }, "ir_version": 10, "name": "MaskedSoftmax/random-023/float32/4x71/seed-20260906", "observation_id": "d012616e76134ed178f50ac43e7ee297ade94bc35b35d09c8c0e54622cff2f4a", "operation": "MaskedSoftmax", "operators": [ "Div", "Equal", "Exp", "ReduceMax", "ReduceSum", "Sub", "Where" ], "opset": 19, "optimization": "disabled", "output_hash": "3ac3cb5e2093db1cd1239117bbc9129a8e07e6de4d059a2a34729f51df9d30c0", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.007709, "p95_ms": 0.0083332, "repeats": 7, "samples_ms": [ 0.008458, 0.008042, 0.0075, 0.007375, 0.00775, 0.007709, 0.006958 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.048417, "p95_ms": 0.051583199999999996, "repeats": 7, "samples_ms": [ 0.051292, 0.049125, 0.048417, 0.048209, 0.048375, 0.051708, 0.048416 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-06, "rtol": 1e-05 }, "variant": "random-023", "warnings": [ { "backend": "reference", "message": "underflow encountered in exp" } ] }, { "artifacts": { "bundle": "observations/ff391cea245186cf69fdb5b2145b5df8f01d5a3c2c3f5894cd2c4601f65797f7/reproducer.zip", "bundle_sha256": "5147d637cf09237ad8c55793fb3d1320819c446e518df10da512885e6e447f50", "case": "artifacts/39937a6649f17f3b513fce18fd264ad491771ef8e9027891c56fc076486cfa67/case.json", "inputs": "artifacts/39937a6649f17f3b513fce18fd264ad491771ef8e9027891c56fc076486cfa67/inputs.npz", "manifest": "artifacts/39937a6649f17f3b513fce18fd264ad491771ef8e9027891c56fc076486cfa67/manifest.json", "model": "artifacts/39937a6649f17f3b513fce18fd264ad491771ef8e9027891c56fc076486cfa67/model.onnx", "outputs": "observations/ff391cea245186cf69fdb5b2145b5df8f01d5a3c2c3f5894cd2c4601f65797f7/outputs.npz" }, "candidate_group": "7920b672048d4018f858715e6684d4a8811ffe17d602b5dab3a488794f908a03", "case_id": "39937a6649f17f3b513fce18fd264ad491771ef8e9027891c56fc076486cfa67", "category": "masking", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "backend": "reference", "name": "probability_range", "passed": true }, { "backend": "reference", "name": "masked_entries_exact_zero", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 0.000244140625, "max_budget_ratio": 0.04438920454545455, "max_rel": 0.000244140625, "max_ulp": 2199023255552, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 8, 17 ], "backend": "reference", "elements": 136, "expected_dtype": "float16", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "backend": "ort", "name": "probability_range", "passed": true }, { "backend": "ort", "name": "masked_entries_exact_zero", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 0.0001220703125, "max_budget_ratio": 0.022194602272727276, "max_rel": 0.0001220703125, "max_ulp": 1099511627776, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 8, 17 ], "backend": "ort", "elements": 136, "expected_dtype": "float16", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } } ], "comparison": { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0001220703125, "max_budget_ratio": 0.08139345596614032, "max_rel": 0.0006105006105006105, "max_ulp": 1, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, "comparison_key": "3467f0ecf0322977bd1b28fb3d81f6010108b2530d970a5526f3b57f45a8a729", "contract": "masked_probability", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "mask": { "dtype": "bool", "shape": [ 4, 17 ] }, "x": { "dtype": "float16", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "MaskedSoftmax/tiny/float16/4x17/seed-20260906", "observation_id": "ff391cea245186cf69fdb5b2145b5df8f01d5a3c2c3f5894cd2c4601f65797f7", "operation": "MaskedSoftmax", "operators": [ "Div", "Equal", "Exp", "ReduceMax", "ReduceSum", "Sub", "Where" ], "opset": 19, "optimization": "all", "output_hash": "61b78065195f2ef8f84864f78034b2497e0f3afd8c945f3b6ff3aee793a027fb", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.008666, "p95_ms": 0.0090168, "repeats": 7, "samples_ms": [ 0.009042, 0.008958, 0.008666, 0.008875, 0.008208, 0.007958, 0.008 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.049375, "p95_ms": 0.0517589, "repeats": 7, "samples_ms": [ 0.051959, 0.050291, 0.051292, 0.049375, 0.048583, 0.048791, 0.048417 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 0.0005, "rtol": 0.005 }, "variant": "tiny", "warnings": [ { "backend": "reference", "message": "underflow encountered in exp" } ] }, { "artifacts": { "bundle": "observations/e82e4ca8a763adceeca59352dbfa27f39931c87cd3e33cf5ab0756b6217d71b9/reproducer.zip", "bundle_sha256": "dc30a5dcf4ebbb22d7c9229562e30567bf4de4be3a96ba1c6758f5d497707718", "case": "artifacts/39937a6649f17f3b513fce18fd264ad491771ef8e9027891c56fc076486cfa67/case.json", "inputs": "artifacts/39937a6649f17f3b513fce18fd264ad491771ef8e9027891c56fc076486cfa67/inputs.npz", "manifest": "artifacts/39937a6649f17f3b513fce18fd264ad491771ef8e9027891c56fc076486cfa67/manifest.json", "model": "artifacts/39937a6649f17f3b513fce18fd264ad491771ef8e9027891c56fc076486cfa67/model.onnx", "outputs": "observations/e82e4ca8a763adceeca59352dbfa27f39931c87cd3e33cf5ab0756b6217d71b9/outputs.npz" }, "candidate_group": "7920b672048d4018f858715e6684d4a8811ffe17d602b5dab3a488794f908a03", "case_id": "39937a6649f17f3b513fce18fd264ad491771ef8e9027891c56fc076486cfa67", "category": "masking", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "backend": "reference", "name": "probability_range", "passed": true }, { "backend": "reference", "name": "masked_entries_exact_zero", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 0.000244140625, "max_budget_ratio": 0.04438920454545455, "max_rel": 0.000244140625, "max_ulp": 2199023255552, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 8, 17 ], "backend": "reference", "elements": 136, "expected_dtype": "float16", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "backend": "ort", "name": "probability_range", "passed": true }, { "backend": "ort", "name": "masked_entries_exact_zero", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 0.0001220703125, "max_budget_ratio": 0.022194602272727276, "max_rel": 0.0001220703125, "max_ulp": 1099511627776, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 8, 17 ], "backend": "ort", "elements": 136, "expected_dtype": "float16", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } } ], "comparison": { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0001220703125, "max_budget_ratio": 0.08139345596614032, "max_rel": 0.0006105006105006105, "max_ulp": 1, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, "comparison_key": "0e67b237b427c92877f7459d62e8942b58dea352bbeadb461d6e01d56035349c", "contract": "masked_probability", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "mask": { "dtype": "bool", "shape": [ 4, 17 ] }, "x": { "dtype": "float16", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "MaskedSoftmax/tiny/float16/4x17/seed-20260906", "observation_id": "e82e4ca8a763adceeca59352dbfa27f39931c87cd3e33cf5ab0756b6217d71b9", "operation": "MaskedSoftmax", "operators": [ "Div", "Equal", "Exp", "ReduceMax", "ReduceSum", "Sub", "Where" ], "opset": 19, "optimization": "disabled", "output_hash": "61b78065195f2ef8f84864f78034b2497e0f3afd8c945f3b6ff3aee793a027fb", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.008792, "p95_ms": 0.0099876, "repeats": 7, "samples_ms": [ 0.010125, 0.009667, 0.009583, 0.008792, 0.008583, 0.008709, 0.0085 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.05075, "p95_ms": 0.0548376, "repeats": 7, "samples_ms": [ 0.05625, 0.051542, 0.050833, 0.0495, 0.049334, 0.049709, 0.05075 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 0.0005, "rtol": 0.005 }, "variant": "tiny", "warnings": [ { "backend": "reference", "message": "underflow encountered in exp" } ] }, { "artifacts": { "bundle": "observations/b8c868b323c15cbaba3e2ecd12088a2a7221e41d5e9675252e303de406b5ae90/reproducer.zip", "bundle_sha256": "a7ca8e9cb15aec44656a327882eb903b1f1a492e3231aa677dd1cddcfe510ecb", "case": "artifacts/8e3f871c141fd36ffcd7d6a4ca69211c67a76e0782ea6cc02b3ba722889ad5de/case.json", "inputs": "artifacts/8e3f871c141fd36ffcd7d6a4ca69211c67a76e0782ea6cc02b3ba722889ad5de/inputs.npz", "manifest": "artifacts/8e3f871c141fd36ffcd7d6a4ca69211c67a76e0782ea6cc02b3ba722889ad5de/manifest.json", "model": "artifacts/8e3f871c141fd36ffcd7d6a4ca69211c67a76e0782ea6cc02b3ba722889ad5de/model.onnx", "outputs": "observations/b8c868b323c15cbaba3e2ecd12088a2a7221e41d5e9675252e303de406b5ae90/outputs.npz" }, "candidate_group": "2bb9887cfbd8bf5f3cc3a256917ccdca233b4ea964f8b8ffe1fa5378f23c0b5a", "case_id": "8e3f871c141fd36ffcd7d6a4ca69211c67a76e0782ea6cc02b3ba722889ad5de", "category": "masking", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "backend": "reference", "name": "probability_range", "passed": true }, { "backend": "reference", "name": "masked_entries_exact_zero", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 8.195638656616211e-08, "max_budget_ratio": 0.007450580596923827, "max_rel": 8.195638656616211e-08, "max_ulp": 369098752, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 17 ], "backend": "reference", "elements": 136, "expected_dtype": "float32", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "backend": "ort", "name": "probability_range", "passed": true }, { "backend": "ort", "name": "masked_entries_exact_zero", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 8.195638656616211e-08, "max_budget_ratio": 0.007450580596923827, "max_rel": 8.195638656616211e-08, "max_ulp": 369098752, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 17 ], "backend": "ort", "elements": 136, "expected_dtype": "float32", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } } ], "comparison": { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, "comparison_key": "2e2c814a59dd7a02cd96e0f771489fad35502d8a97083b47bdf92d130cd38553", "contract": "masked_probability", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "mask": { "dtype": "bool", "shape": [ 4, 17 ] }, "x": { "dtype": "float32", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "MaskedSoftmax/tiny/float32/4x17/seed-20260906", "observation_id": "b8c868b323c15cbaba3e2ecd12088a2a7221e41d5e9675252e303de406b5ae90", "operation": "MaskedSoftmax", "operators": [ "Div", "Equal", "Exp", "ReduceMax", "ReduceSum", "Sub", "Where" ], "opset": 19, "optimization": "all", "output_hash": "62ea9b39b1b85a8b479c348b167f7902918a90498db566fc5aedfe5d03fdd133", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.007625, "p95_ms": 0.0081579, "repeats": 7, "samples_ms": [ 0.008041, 0.008208, 0.00725, 0.007625, 0.007708, 0.007292, 0.006792 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.047334, "p95_ms": 0.0524537, "repeats": 7, "samples_ms": [ 0.053291, 0.048709, 0.0505, 0.047334, 0.047041, 0.04675, 0.046666 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-06, "rtol": 1e-05 }, "variant": "tiny", "warnings": [ { "backend": "reference", "message": "underflow encountered in exp" } ] }, { "artifacts": { "bundle": "observations/46c675ead5d5a8fdb26d502bc5f9b0c0c8a44c46a3ef57a8f2b9bc7627331ba3/reproducer.zip", "bundle_sha256": "ae93a10ba9184052cb20af8e7fdf62041c41deb0b4aa5853a24eaba55aff4b0e", "case": "artifacts/8e3f871c141fd36ffcd7d6a4ca69211c67a76e0782ea6cc02b3ba722889ad5de/case.json", "inputs": "artifacts/8e3f871c141fd36ffcd7d6a4ca69211c67a76e0782ea6cc02b3ba722889ad5de/inputs.npz", "manifest": "artifacts/8e3f871c141fd36ffcd7d6a4ca69211c67a76e0782ea6cc02b3ba722889ad5de/manifest.json", "model": "artifacts/8e3f871c141fd36ffcd7d6a4ca69211c67a76e0782ea6cc02b3ba722889ad5de/model.onnx", "outputs": "observations/46c675ead5d5a8fdb26d502bc5f9b0c0c8a44c46a3ef57a8f2b9bc7627331ba3/outputs.npz" }, "candidate_group": "2bb9887cfbd8bf5f3cc3a256917ccdca233b4ea964f8b8ffe1fa5378f23c0b5a", "case_id": "8e3f871c141fd36ffcd7d6a4ca69211c67a76e0782ea6cc02b3ba722889ad5de", "category": "masking", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "backend": "reference", "name": "probability_range", "passed": true }, { "backend": "reference", "name": "masked_entries_exact_zero", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 8.195638656616211e-08, "max_budget_ratio": 0.007450580596923827, "max_rel": 8.195638656616211e-08, "max_ulp": 369098752, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 17 ], "backend": "reference", "elements": 136, "expected_dtype": "float32", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "backend": "ort", "name": "probability_range", "passed": true }, { "backend": "ort", "name": "masked_entries_exact_zero", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 8.195638656616211e-08, "max_budget_ratio": 0.007450580596923827, "max_rel": 8.195638656616211e-08, "max_ulp": 369098752, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 17 ], "backend": "ort", "elements": 136, "expected_dtype": "float32", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } } ], "comparison": { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, "comparison_key": "f800218b06dca09d39fac35bc70f764ba870772749cb017e6cd43113378e27a3", "contract": "masked_probability", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "mask": { "dtype": "bool", "shape": [ 4, 17 ] }, "x": { "dtype": "float32", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "MaskedSoftmax/tiny/float32/4x17/seed-20260906", "observation_id": "46c675ead5d5a8fdb26d502bc5f9b0c0c8a44c46a3ef57a8f2b9bc7627331ba3", "operation": "MaskedSoftmax", "operators": [ "Div", "Equal", "Exp", "ReduceMax", "ReduceSum", "Sub", "Where" ], "opset": 19, "optimization": "disabled", "output_hash": "62ea9b39b1b85a8b479c348b167f7902918a90498db566fc5aedfe5d03fdd133", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.007167, "p95_ms": 0.007683799999999999, "repeats": 7, "samples_ms": [ 0.007625, 0.007709, 0.007167, 0.007166, 0.007292, 0.006833, 0.006833 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.050667, "p95_ms": 0.055308699999999995, "repeats": 7, "samples_ms": [ 0.051291, 0.050125, 0.048875, 0.050667, 0.047791, 0.056959, 0.051458 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-06, "rtol": 1e-05 }, "variant": "tiny", "warnings": [ { "backend": "reference", "message": "underflow encountered in exp" } ] }, { "artifacts": { "bundle": "observations/549bf76e6f728eb28d9445fc1b90ed981f96b0468b93145554909efd04dd8294/reproducer.zip", "bundle_sha256": "930388756d0c4469b7a4bee4417851fad3a31f3ac38e63537b8144d811778305", "case": "artifacts/dfe22f3d29acafda89e78f94a8b8194d236aecb106e95657f077c647a0f119c6/case.json", "inputs": "artifacts/dfe22f3d29acafda89e78f94a8b8194d236aecb106e95657f077c647a0f119c6/inputs.npz", "manifest": "artifacts/dfe22f3d29acafda89e78f94a8b8194d236aecb106e95657f077c647a0f119c6/manifest.json", "model": "artifacts/dfe22f3d29acafda89e78f94a8b8194d236aecb106e95657f077c647a0f119c6/model.onnx", "outputs": "observations/549bf76e6f728eb28d9445fc1b90ed981f96b0468b93145554909efd04dd8294/outputs.npz" }, "candidate_group": "da2ba9363b8084b3c5e433de8cd9e6fe17146fc24d772db609860ddd1829fe25", "case_id": "dfe22f3d29acafda89e78f94a8b8194d236aecb106e95657f077c647a0f119c6", "category": "masking", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "backend": "reference", "name": "probability_range", "passed": true }, { "backend": "reference", "name": "masked_entries_exact_zero", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 1.1102230246251565e-16, "max_budget_ratio": 0.00010092936587501424, "max_rel": 1.1102230246251565e-16, "max_ulp": 1, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 8, 17 ], "backend": "reference", "elements": 136, "expected_dtype": "float64", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "backend": "ort", "name": "probability_range", "passed": true }, { "backend": "ort", "name": "masked_entries_exact_zero", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 1.1102230246251565e-16, "max_budget_ratio": 0.00010092936587501424, "max_rel": 1.1102230246251565e-16, "max_ulp": 1, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 1.3877787807814457e-17, "max_budget_ratio": 8.738561484061755e-05, "max_rel": 2.3597307139568733e-16, "max_ulp": 2, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 2.7755575615628914e-17, "max_budget_ratio": 0.00014539170032325948, "max_rel": 3.0533493443219357e-16, "max_ulp": 2, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 8, 17 ], "backend": "ort", "elements": 136, "expected_dtype": "float64", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } } ], "comparison": { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 1.3877787807814457e-17, "max_budget_ratio": 8.738561484061754e-05, "max_rel": 2.359730713956873e-16, "max_ulp": 2, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, "comparison_key": "5e37b0b625e050704aa4c540ab299eb32e9cab638ad88b28e5ebe3b56d71aa4d", "contract": "masked_probability", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "mask": { "dtype": "bool", "shape": [ 4, 17 ] }, "x": { "dtype": "float64", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "MaskedSoftmax/tiny/float64/4x17/seed-20260906", "observation_id": "549bf76e6f728eb28d9445fc1b90ed981f96b0468b93145554909efd04dd8294", "operation": "MaskedSoftmax", "operators": [ "Div", "Equal", "Exp", "ReduceMax", "ReduceSum", "Sub", "Where" ], "opset": 19, "optimization": "all", "output_hash": "d7c456706cf5c7b78a1fbcdceb22e1343ea906865030084f934706f53904ba2c", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.0075, "p95_ms": 0.0078249, "repeats": 7, "samples_ms": [ 0.007666, 0.007875, 0.007375, 0.007334, 0.0075, 0.007708, 0.007167 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.048, "p95_ms": 0.0501543, "repeats": 7, "samples_ms": [ 0.050667, 0.048958, 0.0485, 0.047916, 0.047667, 0.048, 0.047958 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-13, "rtol": 1e-12 }, "variant": "tiny", "warnings": [ { "backend": "reference", "message": "underflow encountered in exp" } ] }, { "artifacts": { "bundle": "observations/4f59fb192a23868c59c3228ac08d1137f5a7ea9bcd0c0db6343a6cf6d822cb83/reproducer.zip", "bundle_sha256": "38af43e8b7961106cc6f227a541a0d62a79bfdff85fda41bfd08e4b51ea2aa3a", "case": "artifacts/dfe22f3d29acafda89e78f94a8b8194d236aecb106e95657f077c647a0f119c6/case.json", "inputs": "artifacts/dfe22f3d29acafda89e78f94a8b8194d236aecb106e95657f077c647a0f119c6/inputs.npz", "manifest": "artifacts/dfe22f3d29acafda89e78f94a8b8194d236aecb106e95657f077c647a0f119c6/manifest.json", "model": "artifacts/dfe22f3d29acafda89e78f94a8b8194d236aecb106e95657f077c647a0f119c6/model.onnx", "outputs": "observations/4f59fb192a23868c59c3228ac08d1137f5a7ea9bcd0c0db6343a6cf6d822cb83/outputs.npz" }, "candidate_group": "da2ba9363b8084b3c5e433de8cd9e6fe17146fc24d772db609860ddd1829fe25", "case_id": "dfe22f3d29acafda89e78f94a8b8194d236aecb106e95657f077c647a0f119c6", "category": "masking", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "backend": "reference", "name": "probability_range", "passed": true }, { "backend": "reference", "name": "masked_entries_exact_zero", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 1.1102230246251565e-16, "max_budget_ratio": 0.00010092936587501424, "max_rel": 1.1102230246251565e-16, "max_ulp": 1, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 8, 17 ], "backend": "reference", "elements": 136, "expected_dtype": "float64", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "backend": "ort", "name": "probability_range", "passed": true }, { "backend": "ort", "name": "masked_entries_exact_zero", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 1.1102230246251565e-16, "max_budget_ratio": 0.00010092936587501424, "max_rel": 1.1102230246251565e-16, "max_ulp": 1, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 1.3877787807814457e-17, "max_budget_ratio": 8.738561484061755e-05, "max_rel": 2.3597307139568733e-16, "max_ulp": 2, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 2.7755575615628914e-17, "max_budget_ratio": 0.00014539170032325948, "max_rel": 3.0533493443219357e-16, "max_ulp": 2, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 8, 17 ], "backend": "ort", "elements": 136, "expected_dtype": "float64", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } } ], "comparison": { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 1.3877787807814457e-17, "max_budget_ratio": 8.738561484061754e-05, "max_rel": 2.359730713956873e-16, "max_ulp": 2, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, "comparison_key": "ef1ce1638e9bda7f1aff0245e98daa7d6ef9fcc87d43b2a87c3c4e0ead4cee4d", "contract": "masked_probability", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "mask": { "dtype": "bool", "shape": [ 4, 17 ] }, "x": { "dtype": "float64", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "MaskedSoftmax/tiny/float64/4x17/seed-20260906", "observation_id": "4f59fb192a23868c59c3228ac08d1137f5a7ea9bcd0c0db6343a6cf6d822cb83", "operation": "MaskedSoftmax", "operators": [ "Div", "Equal", "Exp", "ReduceMax", "ReduceSum", "Sub", "Where" ], "opset": 19, "optimization": "disabled", "output_hash": "d7c456706cf5c7b78a1fbcdceb22e1343ea906865030084f934706f53904ba2c", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.00775, "p95_ms": 0.0081999, "repeats": 7, "samples_ms": [ 0.008083, 0.00825, 0.007791, 0.00775, 0.007208, 0.007292, 0.007292 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.049875, "p95_ms": 0.051992, "repeats": 7, "samples_ms": [ 0.052667, 0.050417, 0.049875, 0.049916, 0.048334, 0.048584, 0.048166 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-13, "rtol": 1e-12 }, "variant": "tiny", "warnings": [ { "backend": "reference", "message": "underflow encountered in exp" } ] }, { "artifacts": { "bundle": "observations/f2f73df6429945a0365f02656d0834d3f0278a2983fd94576f351f13f49d2e20/reproducer.zip", "bundle_sha256": "ed2fc6c06d2ebe43e62b084e9aa8f048d1b4f40f87e3fcada0021496c3c3fc37", "case": "artifacts/054a1ad428b1176879063bf3862d5f9ffe2b20b006616219373fd11610328d0f/case.json", "inputs": "artifacts/054a1ad428b1176879063bf3862d5f9ffe2b20b006616219373fd11610328d0f/inputs.npz", "manifest": "artifacts/054a1ad428b1176879063bf3862d5f9ffe2b20b006616219373fd11610328d0f/manifest.json", "model": "artifacts/054a1ad428b1176879063bf3862d5f9ffe2b20b006616219373fd11610328d0f/model.onnx", "outputs": "observations/f2f73df6429945a0365f02656d0834d3f0278a2983fd94576f351f13f49d2e20/outputs.npz" }, "candidate_group": "9c007ea706d4621f76b9e31b822ec0885c2dc5bb7a455a20accb3d60137211a8", "case_id": "054a1ad428b1176879063bf3862d5f9ffe2b20b006616219373fd11610328d0f", "category": "masking", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "backend": "reference", "name": "probability_range", "passed": true }, { "backend": "reference", "name": "masked_entries_exact_zero", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 0.00017213821411132812, "max_budget_ratio": 0.03129785711115057, "max_rel": 0.00017213821411132812, "max_ulp": 775241596928, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 257 ], "backend": "reference", "elements": 1028, "expected_dtype": "float16", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 257 ], "backend": "reference", "elements": 1028, "expected_dtype": "float16", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 257 ], "backend": "reference", "elements": 1028, "expected_dtype": "float16", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 8, 257 ], "backend": "reference", "elements": 2056, "expected_dtype": "float16", "expected_shape": [ 8, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "backend": "ort", "name": "probability_range", "passed": true }, { "backend": "ort", "name": "masked_entries_exact_zero", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 3.337860107421875e-05, "max_budget_ratio": 0.006068836558948864, "max_rel": 3.337860107421875e-05, "max_ulp": 173946175488, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 257 ], "backend": "ort", "elements": 1028, "expected_dtype": "float16", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 257 ], "backend": "ort", "elements": 1028, "expected_dtype": "float16", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 257 ], "backend": "ort", "elements": 1028, "expected_dtype": "float16", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 8, 257 ], "backend": "ort", "elements": 2056, "expected_dtype": "float16", "expected_shape": [ 8, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } } ], "comparison": { "actual_dtype": "float16", "actual_shape": [ 4, 257 ], "elements": 1028, "expected_dtype": "float16", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 3.0517578125e-05, "max_budget_ratio": 0.045789642382892994, "max_rel": 0.0023237800154918666, "max_ulp": 4, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, "comparison_key": "3c762f08c24cdd3ad8c1a2d82d4d7b7b0329729109708abf235e73853f29e742", "contract": "masked_probability", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "mask": { "dtype": "bool", "shape": [ 4, 257 ] }, "x": { "dtype": "float16", "shape": [ 4, 257 ] } }, "ir_version": 10, "name": "MaskedSoftmax/wide/float16/4x257/seed-20260906", "observation_id": "f2f73df6429945a0365f02656d0834d3f0278a2983fd94576f351f13f49d2e20", "operation": "MaskedSoftmax", "operators": [ "Div", "Equal", "Exp", "ReduceMax", "ReduceSum", "Sub", "Where" ], "opset": 19, "optimization": "all", "output_hash": "1f437b9e84de4ad7220541b6edb77ff948ba73b8da3e64875ef7bdf664b9fb48", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.009792, "p95_ms": 0.010462500000000001, "repeats": 7, "samples_ms": [ 0.0105, 0.010375, 0.010083, 0.009792, 0.009708, 0.008917, 0.009542 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.063833, "p95_ms": 0.0683125, "repeats": 7, "samples_ms": [ 0.06925, 0.066125, 0.065208, 0.063833, 0.063792, 0.063333, 0.063375 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 0.0005, "rtol": 0.005 }, "variant": "wide", "warnings": [ { "backend": "reference", "message": "underflow encountered in exp" } ] }, { "artifacts": { "bundle": "observations/f03bb2552b26fcca2c505f12bbd8777e7455a9bbd1a078763fdf3e99d435c0df/reproducer.zip", "bundle_sha256": "dbb0771c3a8bb80dc243df91fa99bbbbaf0f860d6b7f1a96b00d632def7fd2ad", "case": "artifacts/054a1ad428b1176879063bf3862d5f9ffe2b20b006616219373fd11610328d0f/case.json", "inputs": "artifacts/054a1ad428b1176879063bf3862d5f9ffe2b20b006616219373fd11610328d0f/inputs.npz", "manifest": "artifacts/054a1ad428b1176879063bf3862d5f9ffe2b20b006616219373fd11610328d0f/manifest.json", "model": "artifacts/054a1ad428b1176879063bf3862d5f9ffe2b20b006616219373fd11610328d0f/model.onnx", "outputs": "observations/f03bb2552b26fcca2c505f12bbd8777e7455a9bbd1a078763fdf3e99d435c0df/outputs.npz" }, "candidate_group": "9c007ea706d4621f76b9e31b822ec0885c2dc5bb7a455a20accb3d60137211a8", "case_id": "054a1ad428b1176879063bf3862d5f9ffe2b20b006616219373fd11610328d0f", "category": "masking", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "backend": "reference", "name": "probability_range", "passed": true }, { "backend": "reference", "name": "masked_entries_exact_zero", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 0.00017213821411132812, "max_budget_ratio": 0.03129785711115057, "max_rel": 0.00017213821411132812, "max_ulp": 775241596928, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 257 ], "backend": "reference", "elements": 1028, "expected_dtype": "float16", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 257 ], "backend": "reference", "elements": 1028, "expected_dtype": "float16", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 257 ], "backend": "reference", "elements": 1028, "expected_dtype": "float16", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 8, 257 ], "backend": "reference", "elements": 2056, "expected_dtype": "float16", "expected_shape": [ 8, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "backend": "ort", "name": "probability_range", "passed": true }, { "backend": "ort", "name": "masked_entries_exact_zero", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 3.337860107421875e-05, "max_budget_ratio": 0.006068836558948864, "max_rel": 3.337860107421875e-05, "max_ulp": 173946175488, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 257 ], "backend": "ort", "elements": 1028, "expected_dtype": "float16", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 257 ], "backend": "ort", "elements": 1028, "expected_dtype": "float16", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 257 ], "backend": "ort", "elements": 1028, "expected_dtype": "float16", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 8, 257 ], "backend": "ort", "elements": 2056, "expected_dtype": "float16", "expected_shape": [ 8, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } } ], "comparison": { "actual_dtype": "float16", "actual_shape": [ 4, 257 ], "elements": 1028, "expected_dtype": "float16", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 3.0517578125e-05, "max_budget_ratio": 0.045789642382892994, "max_rel": 0.0023237800154918666, "max_ulp": 4, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, "comparison_key": "b2db06f6acf7d7e08f27f19c2eb1d1701d18b09577e71168c03e7378fafd7f4a", "contract": "masked_probability", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "mask": { "dtype": "bool", "shape": [ 4, 257 ] }, "x": { "dtype": "float16", "shape": [ 4, 257 ] } }, "ir_version": 10, "name": "MaskedSoftmax/wide/float16/4x257/seed-20260906", "observation_id": "f03bb2552b26fcca2c505f12bbd8777e7455a9bbd1a078763fdf3e99d435c0df", "operation": "MaskedSoftmax", "operators": [ "Div", "Equal", "Exp", "ReduceMax", "ReduceSum", "Sub", "Where" ], "opset": 19, "optimization": "disabled", "output_hash": "1f437b9e84de4ad7220541b6edb77ff948ba73b8da3e64875ef7bdf664b9fb48", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.009625, "p95_ms": 0.0101414, "repeats": 7, "samples_ms": [ 0.010166, 0.010084, 0.009792, 0.009625, 0.009083, 0.008542, 0.00925 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.06725, "p95_ms": 0.0703162, "repeats": 7, "samples_ms": [ 0.06775, 0.0665, 0.071416, 0.06725, 0.067, 0.06725, 0.065833 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 0.0005, "rtol": 0.005 }, "variant": "wide", "warnings": [ { "backend": "reference", "message": "underflow encountered in exp" } ] }, { "artifacts": { "bundle": "observations/712c8d0fea6a2af13ec668ec100a62ad8412e41ea5150f3c737180e9bdca1b0a/reproducer.zip", "bundle_sha256": "c6636d1743f7c8dc8224efaaba17a3ef61bb13d2ba036967c695101d2e75d3a1", "case": "artifacts/09495c20a247af5be237e289f566d3c7148d8cf2cac691b6422572cefae1fb29/case.json", "inputs": "artifacts/09495c20a247af5be237e289f566d3c7148d8cf2cac691b6422572cefae1fb29/inputs.npz", "manifest": "artifacts/09495c20a247af5be237e289f566d3c7148d8cf2cac691b6422572cefae1fb29/manifest.json", "model": "artifacts/09495c20a247af5be237e289f566d3c7148d8cf2cac691b6422572cefae1fb29/model.onnx", "outputs": "observations/712c8d0fea6a2af13ec668ec100a62ad8412e41ea5150f3c737180e9bdca1b0a/outputs.npz" }, "candidate_group": "44feb0141cbe61c04572790c2e063c37d3c8aee55cec6631e81be161a21a6da1", "case_id": "09495c20a247af5be237e289f566d3c7148d8cf2cac691b6422572cefae1fb29", "category": "masking", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "backend": "reference", "name": "probability_range", "passed": true }, { "backend": "reference", "name": "masked_entries_exact_zero", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 7.3661794885993e-08, "max_budget_ratio": 0.006696526807817545, "max_rel": 7.3661794885993e-08, "max_ulp": 663486464, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 257 ], "backend": "reference", "elements": 1028, "expected_dtype": "float32", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 257 ], "backend": "reference", "elements": 1028, "expected_dtype": "float32", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 257 ], "backend": "reference", "elements": 1028, "expected_dtype": "float32", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 257 ], "backend": "reference", "elements": 2056, "expected_dtype": "float32", "expected_shape": [ 8, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "backend": "ort", "name": "probability_range", "passed": true }, { "backend": "ort", "name": "masked_entries_exact_zero", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 1.214721123687923e-07, "max_budget_ratio": 0.011042919306253843, "max_rel": 1.214721123687923e-07, "max_ulp": 1094123520, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 257 ], "backend": "ort", "elements": 1028, "expected_dtype": "float32", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 3.725290298461914e-09, "max_budget_ratio": 0.0031806633378989213, "max_rel": 2.2384640197128096e-07, "max_ulp": 2, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 257 ], "backend": "ort", "elements": 1028, "expected_dtype": "float32", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 3.725290298461914e-09, "max_budget_ratio": 0.002796285021260826, "max_rel": 1.5745461390436032e-07, "max_ulp": 2, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 257 ], "backend": "ort", "elements": 1028, "expected_dtype": "float32", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 257 ], "backend": "ort", "elements": 2056, "expected_dtype": "float32", "expected_shape": [ 8, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } } ], "comparison": { "actual_dtype": "float32", "actual_shape": [ 4, 257 ], "elements": 1028, "expected_dtype": "float32", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 3.725290298461914e-09, "max_budget_ratio": 0.003153511130645501, "max_rel": 2.2896330908764532e-07, "max_ulp": 3, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, "comparison_key": "d0f8f387612733d2c3ada23220e1cc9b6040278ec50cb2e0912c36a6bed9aa2a", "contract": "masked_probability", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "mask": { "dtype": "bool", "shape": [ 4, 257 ] }, "x": { "dtype": "float32", "shape": [ 4, 257 ] } }, "ir_version": 10, "name": "MaskedSoftmax/wide/float32/4x257/seed-20260906", "observation_id": "712c8d0fea6a2af13ec668ec100a62ad8412e41ea5150f3c737180e9bdca1b0a", "operation": "MaskedSoftmax", "operators": [ "Div", "Equal", "Exp", "ReduceMax", "ReduceSum", "Sub", "Where" ], "opset": 19, "optimization": "all", "output_hash": "b5922da38b39f80763e46f776c8631de1f38410a7f1ad51fa664f6c4436e7672", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.00825, "p95_ms": 0.0090499, "repeats": 7, "samples_ms": [ 0.00925, 0.008583, 0.008458, 0.008167, 0.00825, 0.008042, 0.00775 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.054958, "p95_ms": 0.0572331, "repeats": 7, "samples_ms": [ 0.057708, 0.056125, 0.055458, 0.054958, 0.053417, 0.052875, 0.052542 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-06, "rtol": 1e-05 }, "variant": "wide", "warnings": [ { "backend": "reference", "message": "underflow encountered in exp" } ] }, { "artifacts": { "bundle": "observations/ee5cc96ecae9a829bab6cfc20d901b7d7a8606222c722ef3e8ec461b44fb601b/reproducer.zip", "bundle_sha256": "5f5b2078f3a74ffb42faa5dbb5cabcce3fb030eb0487fc86fd40d6f16188ce29", "case": "artifacts/09495c20a247af5be237e289f566d3c7148d8cf2cac691b6422572cefae1fb29/case.json", "inputs": "artifacts/09495c20a247af5be237e289f566d3c7148d8cf2cac691b6422572cefae1fb29/inputs.npz", "manifest": "artifacts/09495c20a247af5be237e289f566d3c7148d8cf2cac691b6422572cefae1fb29/manifest.json", "model": "artifacts/09495c20a247af5be237e289f566d3c7148d8cf2cac691b6422572cefae1fb29/model.onnx", "outputs": "observations/ee5cc96ecae9a829bab6cfc20d901b7d7a8606222c722ef3e8ec461b44fb601b/outputs.npz" }, "candidate_group": "44feb0141cbe61c04572790c2e063c37d3c8aee55cec6631e81be161a21a6da1", "case_id": "09495c20a247af5be237e289f566d3c7148d8cf2cac691b6422572cefae1fb29", "category": "masking", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "backend": "reference", "name": "probability_range", "passed": true }, { "backend": "reference", "name": "masked_entries_exact_zero", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 7.3661794885993e-08, "max_budget_ratio": 0.006696526807817545, "max_rel": 7.3661794885993e-08, "max_ulp": 663486464, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 257 ], "backend": "reference", "elements": 1028, "expected_dtype": "float32", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 257 ], "backend": "reference", "elements": 1028, "expected_dtype": "float32", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 257 ], "backend": "reference", "elements": 1028, "expected_dtype": "float32", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 257 ], "backend": "reference", "elements": 2056, "expected_dtype": "float32", "expected_shape": [ 8, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "backend": "ort", "name": "probability_range", "passed": true }, { "backend": "ort", "name": "masked_entries_exact_zero", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 1.214721123687923e-07, "max_budget_ratio": 0.011042919306253843, "max_rel": 1.214721123687923e-07, "max_ulp": 1094123520, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 257 ], "backend": "ort", "elements": 1028, "expected_dtype": "float32", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 3.725290298461914e-09, "max_budget_ratio": 0.0031806633378989213, "max_rel": 2.2384640197128096e-07, "max_ulp": 2, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 257 ], "backend": "ort", "elements": 1028, "expected_dtype": "float32", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 3.725290298461914e-09, "max_budget_ratio": 0.002796285021260826, "max_rel": 1.5745461390436032e-07, "max_ulp": 2, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 257 ], "backend": "ort", "elements": 1028, "expected_dtype": "float32", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 257 ], "backend": "ort", "elements": 2056, "expected_dtype": "float32", "expected_shape": [ 8, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } } ], "comparison": { "actual_dtype": "float32", "actual_shape": [ 4, 257 ], "elements": 1028, "expected_dtype": "float32", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 3.725290298461914e-09, "max_budget_ratio": 0.003153511130645501, "max_rel": 2.2896330908764532e-07, "max_ulp": 3, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, "comparison_key": "f11b92f0b69bd7b418c9eb9124969e7e930fa1aa5c02a9026ffa5220cb369570", "contract": "masked_probability", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "mask": { "dtype": "bool", "shape": [ 4, 257 ] }, "x": { "dtype": "float32", "shape": [ 4, 257 ] } }, "ir_version": 10, "name": "MaskedSoftmax/wide/float32/4x257/seed-20260906", "observation_id": "ee5cc96ecae9a829bab6cfc20d901b7d7a8606222c722ef3e8ec461b44fb601b", "operation": "MaskedSoftmax", "operators": [ "Div", "Equal", "Exp", "ReduceMax", "ReduceSum", "Sub", "Where" ], "opset": 19, "optimization": "disabled", "output_hash": "b5922da38b39f80763e46f776c8631de1f38410a7f1ad51fa664f6c4436e7672", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.008208, "p95_ms": 0.0089544, "repeats": 7, "samples_ms": [ 0.00875, 0.009042, 0.008584, 0.007958, 0.008208, 0.008042, 0.007917 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.053917, "p95_ms": 0.0551663, "repeats": 7, "samples_ms": [ 0.055541, 0.05425, 0.053333, 0.052833, 0.053917, 0.054292, 0.053208 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-06, "rtol": 1e-05 }, "variant": "wide", "warnings": [ { "backend": "reference", "message": "underflow encountered in exp" } ] }, { "artifacts": { "bundle": "observations/b6d03c8c0140ea2fb996da08ef74e4073e282c4344ce9c8166c5d10fe3d25a76/reproducer.zip", "bundle_sha256": "f91f78ec0c3c3026b1776a369ea53d31234b818d2090c930498f0e562f9602e7", "case": "artifacts/7a76c32e35f076e36f58f855eb235fc90fb8a23d5186efe81e50667a84c9d936/case.json", "inputs": "artifacts/7a76c32e35f076e36f58f855eb235fc90fb8a23d5186efe81e50667a84c9d936/inputs.npz", "manifest": "artifacts/7a76c32e35f076e36f58f855eb235fc90fb8a23d5186efe81e50667a84c9d936/manifest.json", "model": "artifacts/7a76c32e35f076e36f58f855eb235fc90fb8a23d5186efe81e50667a84c9d936/model.onnx", "outputs": "observations/b6d03c8c0140ea2fb996da08ef74e4073e282c4344ce9c8166c5d10fe3d25a76/outputs.npz" }, "candidate_group": "f5c005cb2ad841c4147d37039cd48cd95142efc535211eaf7b01fa31a33d2a1f", "case_id": "7a76c32e35f076e36f58f855eb235fc90fb8a23d5186efe81e50667a84c9d936", "category": "masking", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "backend": "reference", "name": "probability_range", "passed": true }, { "backend": "reference", "name": "masked_entries_exact_zero", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 1.1102230246251565e-16, "max_budget_ratio": 0.00010092936587501424, "max_rel": 1.1102230246251565e-16, "max_ulp": 1, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 257 ], "backend": "reference", "elements": 1028, "expected_dtype": "float64", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 257 ], "backend": "reference", "elements": 1028, "expected_dtype": "float64", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 257 ], "backend": "reference", "elements": 1028, "expected_dtype": "float64", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 8, 257 ], "backend": "reference", "elements": 2056, "expected_dtype": "float64", "expected_shape": [ 8, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "backend": "ort", "name": "probability_range", "passed": true }, { "backend": "ort", "name": "masked_entries_exact_zero", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 2.220446049250313e-16, "max_budget_ratio": 0.00020185873175002848, "max_rel": 2.220446049250313e-16, "max_ulp": 1, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 257 ], "backend": "ort", "elements": 1028, "expected_dtype": "float64", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 1.3877787807814457e-17, "max_budget_ratio": 8.783148877448103e-05, "max_rel": 3.750980142755907e-16, "max_ulp": 2, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 257 ], "backend": "ort", "elements": 1028, "expected_dtype": "float64", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 1.3877787807814457e-17, "max_budget_ratio": 8.783148877448103e-05, "max_rel": 3.750980142755907e-16, "max_ulp": 2, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 257 ], "backend": "ort", "elements": 1028, "expected_dtype": "float64", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 8, 257 ], "backend": "ort", "elements": 2056, "expected_dtype": "float64", "expected_shape": [ 8, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } } ], "comparison": { "actual_dtype": "float64", "actual_shape": [ 4, 257 ], "elements": 1028, "expected_dtype": "float64", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 1.3877787807814457e-17, "max_budget_ratio": 8.783148877448103e-05, "max_rel": 5.01781029812777e-16, "max_ulp": 3, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, "comparison_key": "7e6a52b486108a5f55d92f59a62d3a5dac6691b1e921c071076d3e6bd49df835", "contract": "masked_probability", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "mask": { "dtype": "bool", "shape": [ 4, 257 ] }, "x": { "dtype": "float64", "shape": [ 4, 257 ] } }, "ir_version": 10, "name": "MaskedSoftmax/wide/float64/4x257/seed-20260906", "observation_id": "b6d03c8c0140ea2fb996da08ef74e4073e282c4344ce9c8166c5d10fe3d25a76", "operation": "MaskedSoftmax", "operators": [ "Div", "Equal", "Exp", "ReduceMax", "ReduceSum", "Sub", "Where" ], "opset": 19, "optimization": "all", "output_hash": "dd8cf05c64637a23a473359c123aaa6dc3ec6220947f9c1cd6248ea00cd1fcac", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.010167, "p95_ms": 0.0111661, "repeats": 7, "samples_ms": [ 0.010583, 0.0105, 0.011416, 0.010125, 0.010167, 0.009875, 0.00975 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.055208, "p95_ms": 0.0593248, "repeats": 7, "samples_ms": [ 0.057458, 0.0595, 0.058916, 0.055208, 0.053917, 0.054334, 0.054958 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-13, "rtol": 1e-12 }, "variant": "wide", "warnings": [ { "backend": "reference", "message": "underflow encountered in exp" } ] }, { "artifacts": { "bundle": "observations/ec17aef5e3b28e2ce74be72575f57fdfc44e98664d414369bf095b94d094b0ff/reproducer.zip", "bundle_sha256": "1a9324c9b0d24fc1d99fac9a5a3ffe464b6f7d1ce4905cc6030321be40e78fb4", "case": "artifacts/7a76c32e35f076e36f58f855eb235fc90fb8a23d5186efe81e50667a84c9d936/case.json", "inputs": "artifacts/7a76c32e35f076e36f58f855eb235fc90fb8a23d5186efe81e50667a84c9d936/inputs.npz", "manifest": "artifacts/7a76c32e35f076e36f58f855eb235fc90fb8a23d5186efe81e50667a84c9d936/manifest.json", "model": "artifacts/7a76c32e35f076e36f58f855eb235fc90fb8a23d5186efe81e50667a84c9d936/model.onnx", "outputs": "observations/ec17aef5e3b28e2ce74be72575f57fdfc44e98664d414369bf095b94d094b0ff/outputs.npz" }, "candidate_group": "f5c005cb2ad841c4147d37039cd48cd95142efc535211eaf7b01fa31a33d2a1f", "case_id": "7a76c32e35f076e36f58f855eb235fc90fb8a23d5186efe81e50667a84c9d936", "category": "masking", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "backend": "reference", "name": "probability_range", "passed": true }, { "backend": "reference", "name": "masked_entries_exact_zero", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 1.1102230246251565e-16, "max_budget_ratio": 0.00010092936587501424, "max_rel": 1.1102230246251565e-16, "max_ulp": 1, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 257 ], "backend": "reference", "elements": 1028, "expected_dtype": "float64", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 257 ], "backend": "reference", "elements": 1028, "expected_dtype": "float64", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 257 ], "backend": "reference", "elements": 1028, "expected_dtype": "float64", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 8, 257 ], "backend": "reference", "elements": 2056, "expected_dtype": "float64", "expected_shape": [ 8, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "backend": "ort", "name": "probability_range", "passed": true }, { "backend": "ort", "name": "masked_entries_exact_zero", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 2.220446049250313e-16, "max_budget_ratio": 0.00020185873175002848, "max_rel": 2.220446049250313e-16, "max_ulp": 1, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 257 ], "backend": "ort", "elements": 1028, "expected_dtype": "float64", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 1.3877787807814457e-17, "max_budget_ratio": 8.783148877448103e-05, "max_rel": 3.750980142755907e-16, "max_ulp": 2, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 257 ], "backend": "ort", "elements": 1028, "expected_dtype": "float64", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 1.3877787807814457e-17, "max_budget_ratio": 8.783148877448103e-05, "max_rel": 3.750980142755907e-16, "max_ulp": 2, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 257 ], "backend": "ort", "elements": 1028, "expected_dtype": "float64", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 8, 257 ], "backend": "ort", "elements": 2056, "expected_dtype": "float64", "expected_shape": [ 8, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } } ], "comparison": { "actual_dtype": "float64", "actual_shape": [ 4, 257 ], "elements": 1028, "expected_dtype": "float64", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 1.3877787807814457e-17, "max_budget_ratio": 8.783148877448103e-05, "max_rel": 5.01781029812777e-16, "max_ulp": 3, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, "comparison_key": "a7d8b9e23e75211b6b3bf9843fa9682a0b2a3cb4df2298a9b6fb6adf244fe404", "contract": "masked_probability", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "mask": { "dtype": "bool", "shape": [ 4, 257 ] }, "x": { "dtype": "float64", "shape": [ 4, 257 ] } }, "ir_version": 10, "name": "MaskedSoftmax/wide/float64/4x257/seed-20260906", "observation_id": "ec17aef5e3b28e2ce74be72575f57fdfc44e98664d414369bf095b94d094b0ff", "operation": "MaskedSoftmax", "operators": [ "Div", "Equal", "Exp", "ReduceMax", "ReduceSum", "Sub", "Where" ], "opset": 19, "optimization": "disabled", "output_hash": "dd8cf05c64637a23a473359c123aaa6dc3ec6220947f9c1cd6248ea00cd1fcac", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.010083, "p95_ms": 0.0108125, "repeats": 7, "samples_ms": [ 0.010375, 0.010083, 0.011, 0.010208, 0.009917, 0.009458, 0.009709 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.055208, "p95_ms": 0.0582498, "repeats": 7, "samples_ms": [ 0.056791, 0.055959, 0.055208, 0.054583, 0.053917, 0.054542, 0.058875 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-13, "rtol": 1e-12 }, "variant": "wide", "warnings": [ { "backend": "reference", "message": "underflow encountered in exp" } ] }, { "artifacts": { "bundle": "observations/d3d41fc7c7feb7c781db6aada00a180dc36a1b03dd154e7d37f6e4f293703960/reproducer.zip", "bundle_sha256": "1a4fbbe86aae654a715b107dcf226e4f5a8cecdaf396d552fd3602d5b79ceabe", "case": "artifacts/3ff16a4db3c88c9ee18c5f248945355ab5ab9e04e415e73f5bf6afbe9df027b5/case.json", "inputs": "artifacts/3ff16a4db3c88c9ee18c5f248945355ab5ab9e04e415e73f5bf6afbe9df027b5/inputs.npz", "manifest": "artifacts/3ff16a4db3c88c9ee18c5f248945355ab5ab9e04e415e73f5bf6afbe9df027b5/manifest.json", "model": "artifacts/3ff16a4db3c88c9ee18c5f248945355ab5ab9e04e415e73f5bf6afbe9df027b5/model.onnx", "outputs": "observations/d3d41fc7c7feb7c781db6aada00a180dc36a1b03dd154e7d37f6e4f293703960/outputs.npz" }, "candidate_group": "44e250d066fba4be7ec63d2271c41f41e14733b9fbb0d750b90e4f0042c133d7", "case_id": "3ff16a4db3c88c9ee18c5f248945355ab5ab9e04e415e73f5bf6afbe9df027b5", "category": "softmax", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "backend": "reference", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 0.000244140625, "max_budget_ratio": 0.04438920454545455, "max_rel": 0.000244140625, "max_ulp": 1099511627776, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 8, 17 ], "backend": "reference", "elements": 136, "expected_dtype": "float16", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "backend": "ort", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 0.000244140625, "max_budget_ratio": 0.04438920454545455, "max_rel": 0.000244140625, "max_ulp": 1099511627776, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 8, 17 ], "backend": "ort", "elements": 136, "expected_dtype": "float16", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } } ], "comparison": { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, "comparison_key": "c1cf93e7bc864cc57c7d46430ea770ce8e08d2dc207bcc53a80f1745a6f057f7", "contract": "probability", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "float16", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "Softmax/constant/float16/4x17/seed-20260906", "observation_id": "d3d41fc7c7feb7c781db6aada00a180dc36a1b03dd154e7d37f6e4f293703960", "operation": "Softmax", "operators": [ "Softmax" ], "opset": 19, "optimization": "all", "output_hash": "8a2cde364af2076060f07a733e0cd55e8972f951ff6b26e2204372de064b1e5b", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.004875, "p95_ms": 0.0052965, "repeats": 7, "samples_ms": [ 0.005334, 0.005209, 0.004875, 0.004917, 0.004584, 0.004291, 0.0045 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.008875, "p95_ms": 0.0095829, "repeats": 7, "samples_ms": [ 0.009708, 0.009291, 0.008875, 0.00875, 0.009084, 0.008875, 0.008459 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 0.0005, "rtol": 0.005 }, "variant": "constant", "warnings": [] }, { "artifacts": { "bundle": "observations/c9f3333792f3d866086769f21e492ceff121607980aafce0e2364dfed833f275/reproducer.zip", "bundle_sha256": "88a45ac291189d9a8cf49e956ce008c02da71530a92d3ab1fa2a3684b218e128", "case": "artifacts/3ff16a4db3c88c9ee18c5f248945355ab5ab9e04e415e73f5bf6afbe9df027b5/case.json", "inputs": "artifacts/3ff16a4db3c88c9ee18c5f248945355ab5ab9e04e415e73f5bf6afbe9df027b5/inputs.npz", "manifest": "artifacts/3ff16a4db3c88c9ee18c5f248945355ab5ab9e04e415e73f5bf6afbe9df027b5/manifest.json", "model": "artifacts/3ff16a4db3c88c9ee18c5f248945355ab5ab9e04e415e73f5bf6afbe9df027b5/model.onnx", "outputs": "observations/c9f3333792f3d866086769f21e492ceff121607980aafce0e2364dfed833f275/outputs.npz" }, "candidate_group": "44e250d066fba4be7ec63d2271c41f41e14733b9fbb0d750b90e4f0042c133d7", "case_id": "3ff16a4db3c88c9ee18c5f248945355ab5ab9e04e415e73f5bf6afbe9df027b5", "category": "softmax", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "backend": "reference", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 0.000244140625, "max_budget_ratio": 0.04438920454545455, "max_rel": 0.000244140625, "max_ulp": 1099511627776, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 8, 17 ], "backend": "reference", "elements": 136, "expected_dtype": "float16", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "backend": "ort", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 0.000244140625, "max_budget_ratio": 0.04438920454545455, "max_rel": 0.000244140625, "max_ulp": 1099511627776, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 8, 17 ], "backend": "ort", "elements": 136, "expected_dtype": "float16", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } } ], "comparison": { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, "comparison_key": "338524b4b8cad25a25ce73731bbd37b025f23e31ec39bfd59f0cf549e4e102b8", "contract": "probability", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "float16", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "Softmax/constant/float16/4x17/seed-20260906", "observation_id": "c9f3333792f3d866086769f21e492ceff121607980aafce0e2364dfed833f275", "operation": "Softmax", "operators": [ "Softmax" ], "opset": 19, "optimization": "disabled", "output_hash": "8a2cde364af2076060f07a733e0cd55e8972f951ff6b26e2204372de064b1e5b", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.004959, "p95_ms": 0.005967, "repeats": 7, "samples_ms": [ 0.006042, 0.005792, 0.005291, 0.004959, 0.004708, 0.004542, 0.004708 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.009041, "p95_ms": 0.0099501, "repeats": 7, "samples_ms": [ 0.010125, 0.009542, 0.009125, 0.009041, 0.00875, 0.008792, 0.008667 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 0.0005, "rtol": 0.005 }, "variant": "constant", "warnings": [] }, { "artifacts": { "bundle": "observations/8ae92cd4f14b48aa1be453fea7b6f4d51d10ea6ece2685e40778e58925fbb89c/reproducer.zip", "bundle_sha256": "a618637fef5f2f08d5e8eabf4f9e7d0569e61ab7d92d9b8ea7294161c0227c59", "case": "artifacts/ccfb2f7d31b5d00ad5478735613b2aadf80ef23349e4421ab8a8292eb3e47e6e/case.json", "inputs": "artifacts/ccfb2f7d31b5d00ad5478735613b2aadf80ef23349e4421ab8a8292eb3e47e6e/inputs.npz", "manifest": "artifacts/ccfb2f7d31b5d00ad5478735613b2aadf80ef23349e4421ab8a8292eb3e47e6e/manifest.json", "model": "artifacts/ccfb2f7d31b5d00ad5478735613b2aadf80ef23349e4421ab8a8292eb3e47e6e/model.onnx", "outputs": "observations/8ae92cd4f14b48aa1be453fea7b6f4d51d10ea6ece2685e40778e58925fbb89c/outputs.npz" }, "candidate_group": "6e97e627526df7e7ab558073b1dccfbf767861bf02392fc0285040710b0d1a4e", "case_id": "ccfb2f7d31b5d00ad5478735613b2aadf80ef23349e4421ab8a8292eb3e47e6e", "category": "softmax", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "backend": "reference", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 3.725290298461914e-09, "max_budget_ratio": 0.0003386627544056285, "max_rel": 3.725290298461914e-09, "max_ulp": 16777216, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 17 ], "backend": "reference", "elements": 136, "expected_dtype": "float32", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "backend": "ort", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 3.725290298461914e-09, "max_budget_ratio": 0.0003386627544056285, "max_rel": 3.725290298461914e-09, "max_ulp": 16777216, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 17 ], "backend": "ort", "elements": 136, "expected_dtype": "float32", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } } ], "comparison": { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, "comparison_key": "3b09c86520f20624e704d392845e278507fdf199655561c6be0fa7eeca2d2f8d", "contract": "probability", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "float32", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "Softmax/constant/float32/4x17/seed-20260906", "observation_id": "8ae92cd4f14b48aa1be453fea7b6f4d51d10ea6ece2685e40778e58925fbb89c", "operation": "Softmax", "operators": [ "Softmax" ], "opset": 19, "optimization": "all", "output_hash": "4744852e52bd33335ed2c60bc090f957605ef873b3cf71f064b0c5393ddc5d5d", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.002875, "p95_ms": 0.0032830999999999997, "repeats": 7, "samples_ms": [ 0.002875, 0.003458, 0.002875, 0.00275, 0.002792, 0.0025, 0.002875 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.008333, "p95_ms": 0.009237599999999999, "repeats": 7, "samples_ms": [ 0.009375, 0.008917, 0.008459, 0.008333, 0.008167, 0.008125, 0.008 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-06, "rtol": 1e-05 }, "variant": "constant", "warnings": [] }, { "artifacts": { "bundle": "observations/50b2c7ea71e63afe0bb471c99b2f71169c5ac35ff6dbc9bddfa0348c5ff3a181/reproducer.zip", "bundle_sha256": "0dca119804678e4067e3b476f7df006e573c77cec539db9fff522d6d3ce0acff", "case": "artifacts/ccfb2f7d31b5d00ad5478735613b2aadf80ef23349e4421ab8a8292eb3e47e6e/case.json", "inputs": "artifacts/ccfb2f7d31b5d00ad5478735613b2aadf80ef23349e4421ab8a8292eb3e47e6e/inputs.npz", "manifest": "artifacts/ccfb2f7d31b5d00ad5478735613b2aadf80ef23349e4421ab8a8292eb3e47e6e/manifest.json", "model": "artifacts/ccfb2f7d31b5d00ad5478735613b2aadf80ef23349e4421ab8a8292eb3e47e6e/model.onnx", "outputs": "observations/50b2c7ea71e63afe0bb471c99b2f71169c5ac35ff6dbc9bddfa0348c5ff3a181/outputs.npz" }, "candidate_group": "6e97e627526df7e7ab558073b1dccfbf767861bf02392fc0285040710b0d1a4e", "case_id": "ccfb2f7d31b5d00ad5478735613b2aadf80ef23349e4421ab8a8292eb3e47e6e", "category": "softmax", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "backend": "reference", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 3.725290298461914e-09, "max_budget_ratio": 0.0003386627544056285, "max_rel": 3.725290298461914e-09, "max_ulp": 16777216, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 17 ], "backend": "reference", "elements": 136, "expected_dtype": "float32", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "backend": "ort", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 3.725290298461914e-09, "max_budget_ratio": 0.0003386627544056285, "max_rel": 3.725290298461914e-09, "max_ulp": 16777216, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 17 ], "backend": "ort", "elements": 136, "expected_dtype": "float32", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } } ], "comparison": { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, "comparison_key": "c08484042e00e22a1abec3599565efb0613f645ca66a3635fc85d9152fbcbf5f", "contract": "probability", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "float32", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "Softmax/constant/float32/4x17/seed-20260906", "observation_id": "50b2c7ea71e63afe0bb471c99b2f71169c5ac35ff6dbc9bddfa0348c5ff3a181", "operation": "Softmax", "operators": [ "Softmax" ], "opset": 19, "optimization": "disabled", "output_hash": "4744852e52bd33335ed2c60bc090f957605ef873b3cf71f064b0c5393ddc5d5d", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.002792, "p95_ms": 0.0032747999999999996, "repeats": 7, "samples_ms": [ 0.002917, 0.003375, 0.003041, 0.002792, 0.002709, 0.002792, 0.0025 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.008333, "p95_ms": 0.009349999999999999, "repeats": 7, "samples_ms": [ 0.0095, 0.009, 0.008458, 0.008333, 0.008167, 0.008042, 0.008 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-06, "rtol": 1e-05 }, "variant": "constant", "warnings": [] }, { "artifacts": { "bundle": "observations/2eec153ed38a70300dd21350e579d6fd15e90c020585d35b97ba81ced7c9d298/reproducer.zip", "bundle_sha256": "38b0d5125cf743342e6973baa2ee50e7f2713e11f28954b176fc451a535f3fc3", "case": "artifacts/ea29959966526ac60847ebb22fa449ce41202dd856585c6abd8aa986683b0f52/case.json", "inputs": "artifacts/ea29959966526ac60847ebb22fa449ce41202dd856585c6abd8aa986683b0f52/inputs.npz", "manifest": "artifacts/ea29959966526ac60847ebb22fa449ce41202dd856585c6abd8aa986683b0f52/manifest.json", "model": "artifacts/ea29959966526ac60847ebb22fa449ce41202dd856585c6abd8aa986683b0f52/model.onnx", "outputs": "observations/2eec153ed38a70300dd21350e579d6fd15e90c020585d35b97ba81ced7c9d298/outputs.npz" }, "candidate_group": "5e90ce2b6c57880c3d8b0ca9737c0b349e5658a4df712c456907c09a961b9767", "case_id": "ea29959966526ac60847ebb22fa449ce41202dd856585c6abd8aa986683b0f52", "category": "softmax", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "backend": "reference", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 8, 17 ], "backend": "reference", "elements": 136, "expected_dtype": "float64", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "backend": "ort", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 8, 17 ], "backend": "ort", "elements": 136, "expected_dtype": "float64", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } } ], "comparison": { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, "comparison_key": "dda4badf57704b9137033acff9796d752080ec30158e5963ffa1e163e66e3d5c", "contract": "probability", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "float64", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "Softmax/constant/float64/4x17/seed-20260906", "observation_id": "2eec153ed38a70300dd21350e579d6fd15e90c020585d35b97ba81ced7c9d298", "operation": "Softmax", "operators": [ "Softmax" ], "opset": 19, "optimization": "all", "output_hash": "394404ee793bbeb7f31db07dc6ccea7527c8c5cb0d85041206f323f867584ac7", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.003208, "p95_ms": 0.003958199999999999, "repeats": 7, "samples_ms": [ 0.003583, 0.004083, 0.003667, 0.003208, 0.003208, 0.003042, 0.003 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.0085, "p95_ms": 0.0091501, "repeats": 7, "samples_ms": [ 0.00925, 0.008917, 0.0085, 0.00825, 0.008208, 0.008584, 0.008292 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-13, "rtol": 1e-12 }, "variant": "constant", "warnings": [] }, { "artifacts": { "bundle": "observations/68b8bd2d8697bce7d44da6a55a0917f3553e14e3800a93344b8c0381063e82df/reproducer.zip", "bundle_sha256": "76cdfa0c323d1cca48ff2df989a06ba9b15de96ec2265d0e24477a1e03fe439e", "case": "artifacts/ea29959966526ac60847ebb22fa449ce41202dd856585c6abd8aa986683b0f52/case.json", "inputs": "artifacts/ea29959966526ac60847ebb22fa449ce41202dd856585c6abd8aa986683b0f52/inputs.npz", "manifest": "artifacts/ea29959966526ac60847ebb22fa449ce41202dd856585c6abd8aa986683b0f52/manifest.json", "model": "artifacts/ea29959966526ac60847ebb22fa449ce41202dd856585c6abd8aa986683b0f52/model.onnx", "outputs": "observations/68b8bd2d8697bce7d44da6a55a0917f3553e14e3800a93344b8c0381063e82df/outputs.npz" }, "candidate_group": "5e90ce2b6c57880c3d8b0ca9737c0b349e5658a4df712c456907c09a961b9767", "case_id": "ea29959966526ac60847ebb22fa449ce41202dd856585c6abd8aa986683b0f52", "category": "softmax", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "backend": "reference", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 8, 17 ], "backend": "reference", "elements": 136, "expected_dtype": "float64", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "backend": "ort", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 8, 17 ], "backend": "ort", "elements": 136, "expected_dtype": "float64", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } } ], "comparison": { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, "comparison_key": "e41c2333b8c8de7d519584524bb9983a04d635470c9259b28d570e838d773279", "contract": "probability", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "float64", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "Softmax/constant/float64/4x17/seed-20260906", "observation_id": "68b8bd2d8697bce7d44da6a55a0917f3553e14e3800a93344b8c0381063e82df", "operation": "Softmax", "operators": [ "Softmax" ], "opset": 19, "optimization": "disabled", "output_hash": "394404ee793bbeb7f31db07dc6ccea7527c8c5cb0d85041206f323f867584ac7", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.003542, "p95_ms": 0.0037876999999999997, "repeats": 7, "samples_ms": [ 0.003584, 0.003875, 0.003542, 0.003583, 0.00325, 0.003042, 0.003167 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.008166, "p95_ms": 0.0090376, "repeats": 7, "samples_ms": [ 0.00925, 0.008542, 0.008209, 0.008041, 0.008042, 0.008166, 0.007792 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-13, "rtol": 1e-12 }, "variant": "constant", "warnings": [] }, { "artifacts": { "bundle": "observations/10332f0bfaf701eec9874b928d7fdd305d1a5449c9951f4750b2e71ee8ca0f14/reproducer.zip", "bundle_sha256": "1598bc8026141c211abc37379a42404900d692b0b5cfa362177b74456d1c7da0", "case": "artifacts/168d8b204c5d76c0b0a6d3c65304950d80545d2f0734c02dff34105284bb415f/case.json", "inputs": "artifacts/168d8b204c5d76c0b0a6d3c65304950d80545d2f0734c02dff34105284bb415f/inputs.npz", "manifest": "artifacts/168d8b204c5d76c0b0a6d3c65304950d80545d2f0734c02dff34105284bb415f/manifest.json", "model": "artifacts/168d8b204c5d76c0b0a6d3c65304950d80545d2f0734c02dff34105284bb415f/model.onnx", "outputs": "observations/10332f0bfaf701eec9874b928d7fdd305d1a5449c9951f4750b2e71ee8ca0f14/outputs.npz" }, "candidate_group": "b00924ae6d67a16142b02dfec203ff3460fc6d00b31e793d970e0ca9e3ad3438", "case_id": "168d8b204c5d76c0b0a6d3c65304950d80545d2f0734c02dff34105284bb415f", "category": "softmax", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "backend": "reference", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 0.000213623046875, "max_budget_ratio": 0.03884055397727273, "max_rel": 0.000213623046875, "max_ulp": 1924145348608, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 8, 17 ], "backend": "reference", "elements": 136, "expected_dtype": "float16", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "backend": "ort", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 9.1552734375e-05, "max_budget_ratio": 0.016645951704545456, "max_rel": 9.1552734375e-05, "max_ulp": 824633720832, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 8, 17 ], "backend": "ort", "elements": 136, "expected_dtype": "float16", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } } ], "comparison": { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 3.0517578125e-05, "max_budget_ratio": 0.040553144896386714, "max_rel": 0.0006042296072507553, "max_ulp": 1, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, "comparison_key": "82bc8dbab6a0c5ce345d3b95f014272fb2e847c7fa3bc580ffdd60aef4b0b7eb", "contract": "probability", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "float16", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "Softmax/large_offset/float16/4x17/seed-20260906", "observation_id": "10332f0bfaf701eec9874b928d7fdd305d1a5449c9951f4750b2e71ee8ca0f14", "operation": "Softmax", "operators": [ "Softmax" ], "opset": 19, "optimization": "all", "output_hash": "f5adfc28512910840037fea185ad82b378a0f54edd3018fb64c9dbadc7ec6019", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.004792, "p95_ms": 0.0056706, "repeats": 7, "samples_ms": [ 0.004959, 0.005, 0.004792, 0.004458, 0.004667, 0.003958, 0.005958 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.010167, "p95_ms": 0.01892489999999999, "repeats": 7, "samples_ms": [ 0.011458, 0.010167, 0.009, 0.008875, 0.010458, 0.010042, 0.022125 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 0.0005, "rtol": 0.005 }, "variant": "large_offset", "warnings": [] }, { "artifacts": { "bundle": "observations/4386560b60bc72aab16a1886fa53fd72bdb48bf88ae8d1ace71f830d06c87a11/reproducer.zip", "bundle_sha256": "98984c666a10096b9e03be4e7503806d0cca229ff9e68e84b979f377fcf29231", "case": "artifacts/168d8b204c5d76c0b0a6d3c65304950d80545d2f0734c02dff34105284bb415f/case.json", "inputs": "artifacts/168d8b204c5d76c0b0a6d3c65304950d80545d2f0734c02dff34105284bb415f/inputs.npz", "manifest": "artifacts/168d8b204c5d76c0b0a6d3c65304950d80545d2f0734c02dff34105284bb415f/manifest.json", "model": "artifacts/168d8b204c5d76c0b0a6d3c65304950d80545d2f0734c02dff34105284bb415f/model.onnx", "outputs": "observations/4386560b60bc72aab16a1886fa53fd72bdb48bf88ae8d1ace71f830d06c87a11/outputs.npz" }, "candidate_group": "b00924ae6d67a16142b02dfec203ff3460fc6d00b31e793d970e0ca9e3ad3438", "case_id": "168d8b204c5d76c0b0a6d3c65304950d80545d2f0734c02dff34105284bb415f", "category": "softmax", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "backend": "reference", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 0.000213623046875, "max_budget_ratio": 0.03884055397727273, "max_rel": 0.000213623046875, "max_ulp": 1924145348608, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 8, 17 ], "backend": "reference", "elements": 136, "expected_dtype": "float16", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "backend": "ort", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 9.1552734375e-05, "max_budget_ratio": 0.016645951704545456, "max_rel": 9.1552734375e-05, "max_ulp": 824633720832, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 8, 17 ], "backend": "ort", "elements": 136, "expected_dtype": "float16", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } } ], "comparison": { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 3.0517578125e-05, "max_budget_ratio": 0.040553144896386714, "max_rel": 0.0006042296072507553, "max_ulp": 1, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, "comparison_key": "f411d9feb2a097835610672f495fb8069954419ac32eb8e3b69c8daf9428f939", "contract": "probability", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "float16", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "Softmax/large_offset/float16/4x17/seed-20260906", "observation_id": "4386560b60bc72aab16a1886fa53fd72bdb48bf88ae8d1ace71f830d06c87a11", "operation": "Softmax", "operators": [ "Softmax" ], "opset": 19, "optimization": "disabled", "output_hash": "f5adfc28512910840037fea185ad82b378a0f54edd3018fb64c9dbadc7ec6019", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.005167, "p95_ms": 0.0058251, "repeats": 7, "samples_ms": [ 0.006, 0.005417, 0.005167, 0.005, 0.004875, 0.004958, 0.005333 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.009625, "p95_ms": 0.010412600000000001, "repeats": 7, "samples_ms": [ 0.010625, 0.009917, 0.009708, 0.009334, 0.00925, 0.009625, 0.009209 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 0.0005, "rtol": 0.005 }, "variant": "large_offset", "warnings": [] }, { "artifacts": { "bundle": "observations/9df8e5c3b60e86e5a7d4069917e8e34e6ef8d14c9e1d8a7347e4743ee8a0eea0/reproducer.zip", "bundle_sha256": "a6fd3bba57843cbbaa725428abc83d405a4064de9b8ebfd57b414e1886f425d9", "case": "artifacts/f91a9aed82f31ad5d7d4251dde15bc08208d9457d4e1c192c346ca03fea20714/case.json", "inputs": "artifacts/f91a9aed82f31ad5d7d4251dde15bc08208d9457d4e1c192c346ca03fea20714/inputs.npz", "manifest": "artifacts/f91a9aed82f31ad5d7d4251dde15bc08208d9457d4e1c192c346ca03fea20714/manifest.json", "model": "artifacts/f91a9aed82f31ad5d7d4251dde15bc08208d9457d4e1c192c346ca03fea20714/model.onnx", "outputs": "observations/9df8e5c3b60e86e5a7d4069917e8e34e6ef8d14c9e1d8a7347e4743ee8a0eea0/outputs.npz" }, "candidate_group": "d109a2aa3f2935123ce1633c6486c1f080a37acf4d5a76a4a3127bf3e865374c", "case_id": "f91a9aed82f31ad5d7d4251dde15bc08208d9457d4e1c192c346ca03fea20714", "category": "softmax", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "backend": "reference", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 5.960464477539063e-08, "max_budget_ratio": 0.005418604070490056, "max_rel": 5.960464477539063e-08, "max_ulp": 536870912, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 17 ], "backend": "reference", "elements": 136, "expected_dtype": "float32", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "backend": "ort", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 8.940696716308594e-08, "max_budget_ratio": 0.008127906105735085, "max_rel": 8.940696716308594e-08, "max_ulp": 805306368, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 17 ], "backend": "ort", "elements": 136, "expected_dtype": "float32", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 7.450580596923828e-09, "max_budget_ratio": 0.005000190557262137, "max_rel": 1.5203425635864272e-07, "max_ulp": 2, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } } ], "comparison": { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 7.450580596923828e-09, "max_budget_ratio": 0.0050762246399558225, "max_rel": 1.592887565871874e-07, "max_ulp": 2, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, "comparison_key": "42d3d183f3a31777df703a3de28fa3dcf0c7f2a285716539e0a9c7554e14b4a6", "contract": "probability", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "float32", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "Softmax/large_offset/float32/4x17/seed-20260906", "observation_id": "9df8e5c3b60e86e5a7d4069917e8e34e6ef8d14c9e1d8a7347e4743ee8a0eea0", "operation": "Softmax", "operators": [ "Softmax" ], "opset": 19, "optimization": "all", "output_hash": "3d6181c640aa33000ddb8e7bffd2b6ebb7fe37b2f69e95da53e20a4eca237ef2", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.002958, "p95_ms": 0.0034080999999999994, "repeats": 7, "samples_ms": [ 0.002916, 0.003583, 0.003, 0.003, 0.002917, 0.002958, 0.002709 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.008541, "p95_ms": 0.009, "repeats": 7, "samples_ms": [ 0.009, 0.008875, 0.008333, 0.008458, 0.008541, 0.009, 0.008375 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-06, "rtol": 1e-05 }, "variant": "large_offset", "warnings": [] }, { "artifacts": { "bundle": "observations/47e8b562cd639ec731dd21dd58ed72d5d5950ace2b430b5a1df23cbc1e40f746/reproducer.zip", "bundle_sha256": "4014eaa3ddacc17c0f6d37481fb2a0ef742b35bdc3c42097dc2512cb086b150f", "case": "artifacts/f91a9aed82f31ad5d7d4251dde15bc08208d9457d4e1c192c346ca03fea20714/case.json", "inputs": "artifacts/f91a9aed82f31ad5d7d4251dde15bc08208d9457d4e1c192c346ca03fea20714/inputs.npz", "manifest": "artifacts/f91a9aed82f31ad5d7d4251dde15bc08208d9457d4e1c192c346ca03fea20714/manifest.json", "model": "artifacts/f91a9aed82f31ad5d7d4251dde15bc08208d9457d4e1c192c346ca03fea20714/model.onnx", "outputs": "observations/47e8b562cd639ec731dd21dd58ed72d5d5950ace2b430b5a1df23cbc1e40f746/outputs.npz" }, "candidate_group": "d109a2aa3f2935123ce1633c6486c1f080a37acf4d5a76a4a3127bf3e865374c", "case_id": "f91a9aed82f31ad5d7d4251dde15bc08208d9457d4e1c192c346ca03fea20714", "category": "softmax", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "backend": "reference", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 5.960464477539063e-08, "max_budget_ratio": 0.005418604070490056, "max_rel": 5.960464477539063e-08, "max_ulp": 536870912, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 17 ], "backend": "reference", "elements": 136, "expected_dtype": "float32", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "backend": "ort", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 8.940696716308594e-08, "max_budget_ratio": 0.008127906105735085, "max_rel": 8.940696716308594e-08, "max_ulp": 805306368, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 17 ], "backend": "ort", "elements": 136, "expected_dtype": "float32", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 7.450580596923828e-09, "max_budget_ratio": 0.005000190557262137, "max_rel": 1.5203425635864272e-07, "max_ulp": 2, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } } ], "comparison": { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 7.450580596923828e-09, "max_budget_ratio": 0.0050762246399558225, "max_rel": 1.592887565871874e-07, "max_ulp": 2, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, "comparison_key": "0c78538c41bddd3e4bc425eb77de0d44e19900a3060bcc84a98564416b3ca26e", "contract": "probability", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "float32", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "Softmax/large_offset/float32/4x17/seed-20260906", "observation_id": "47e8b562cd639ec731dd21dd58ed72d5d5950ace2b430b5a1df23cbc1e40f746", "operation": "Softmax", "operators": [ "Softmax" ], "opset": 19, "optimization": "disabled", "output_hash": "3d6181c640aa33000ddb8e7bffd2b6ebb7fe37b2f69e95da53e20a4eca237ef2", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.003041, "p95_ms": 0.0033997999999999997, "repeats": 7, "samples_ms": [ 0.003041, 0.0035, 0.003084, 0.003166, 0.003041, 0.002917, 0.002916 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.008209, "p95_ms": 0.009279599999999999, "repeats": 7, "samples_ms": [ 0.009417, 0.008959, 0.008458, 0.008209, 0.007875, 0.007958, 0.007959 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-06, "rtol": 1e-05 }, "variant": "large_offset", "warnings": [] }, { "artifacts": { "bundle": "observations/db6c4b5a8027fefc9337cec95f8aa9c7ec2c83c3d4d41baf4d30ff9295f7671c/reproducer.zip", "bundle_sha256": "cb62c45a50e75dd0b86095507199ef290b5d91e68455e67c84c00688f7d873ed", "case": "artifacts/ad779dfbe8dd2f7fcb66205d8f25effe36d2d0a6ba9d8d2baec2e326cf93e2ea/case.json", "inputs": "artifacts/ad779dfbe8dd2f7fcb66205d8f25effe36d2d0a6ba9d8d2baec2e326cf93e2ea/inputs.npz", "manifest": "artifacts/ad779dfbe8dd2f7fcb66205d8f25effe36d2d0a6ba9d8d2baec2e326cf93e2ea/manifest.json", "model": "artifacts/ad779dfbe8dd2f7fcb66205d8f25effe36d2d0a6ba9d8d2baec2e326cf93e2ea/model.onnx", "outputs": "observations/db6c4b5a8027fefc9337cec95f8aa9c7ec2c83c3d4d41baf4d30ff9295f7671c/outputs.npz" }, "candidate_group": "792d715ffc154462222c0ce49cc866064c1f717d854b27b0a426ecca4df7f0cf", "case_id": "ad779dfbe8dd2f7fcb66205d8f25effe36d2d0a6ba9d8d2baec2e326cf93e2ea", "category": "softmax", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "backend": "reference", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 1.1102230246251565e-16, "max_budget_ratio": 0.00010092936587501424, "max_rel": 1.1102230246251565e-16, "max_ulp": 1, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 8, 17 ], "backend": "reference", "elements": 136, "expected_dtype": "float64", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "backend": "ort", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 2.220446049250313e-16, "max_budget_ratio": 0.00020185873175002848, "max_rel": 2.220446049250313e-16, "max_ulp": 2, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 1.3877787807814457e-17, "max_budget_ratio": 8.596324118225444e-05, "max_rel": 2.2588049270336617e-16, "max_ulp": 2, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 8, 17 ], "backend": "ort", "elements": 136, "expected_dtype": "float64", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 2.7755575615628914e-17, "max_budget_ratio": 0.0001650553953654569, "max_rel": 4.072171390944368e-16, "max_ulp": 3, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } } ], "comparison": { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 2.7755575615628914e-17, "max_budget_ratio": 0.00016505539536545687, "max_rel": 4.0721713909443665e-16, "max_ulp": 3, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, "comparison_key": "6941cce891ba2c7240b2d571baba1da9accf46ea0f45459dc00209d4b883632d", "contract": "probability", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "float64", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "Softmax/large_offset/float64/4x17/seed-20260906", "observation_id": "db6c4b5a8027fefc9337cec95f8aa9c7ec2c83c3d4d41baf4d30ff9295f7671c", "operation": "Softmax", "operators": [ "Softmax" ], "opset": 19, "optimization": "all", "output_hash": "916886b290c95e6d9c2b3f510f548b3857d6e04554007caacab78ec928f83a79", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.003208, "p95_ms": 0.0037668, "repeats": 7, "samples_ms": [ 0.003208, 0.003792, 0.003708, 0.003208, 0.003125, 0.003125, 0.003166 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.008125, "p95_ms": 0.009058299999999998, "repeats": 7, "samples_ms": [ 0.009208, 0.008709, 0.008417, 0.008125, 0.008042, 0.008125, 0.007875 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-13, "rtol": 1e-12 }, "variant": "large_offset", "warnings": [] }, { "artifacts": { "bundle": "observations/83387ced5d1c7ec2241f7816fb9ea04a0d06454398c484222434516d963144c7/reproducer.zip", "bundle_sha256": "5bb4c03adf8b5292a9619cab43b62dd9c187124d772d287000fcc2610b3b416c", "case": "artifacts/ad779dfbe8dd2f7fcb66205d8f25effe36d2d0a6ba9d8d2baec2e326cf93e2ea/case.json", "inputs": "artifacts/ad779dfbe8dd2f7fcb66205d8f25effe36d2d0a6ba9d8d2baec2e326cf93e2ea/inputs.npz", "manifest": "artifacts/ad779dfbe8dd2f7fcb66205d8f25effe36d2d0a6ba9d8d2baec2e326cf93e2ea/manifest.json", "model": "artifacts/ad779dfbe8dd2f7fcb66205d8f25effe36d2d0a6ba9d8d2baec2e326cf93e2ea/model.onnx", "outputs": "observations/83387ced5d1c7ec2241f7816fb9ea04a0d06454398c484222434516d963144c7/outputs.npz" }, "candidate_group": "792d715ffc154462222c0ce49cc866064c1f717d854b27b0a426ecca4df7f0cf", "case_id": "ad779dfbe8dd2f7fcb66205d8f25effe36d2d0a6ba9d8d2baec2e326cf93e2ea", "category": "softmax", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "backend": "reference", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 1.1102230246251565e-16, "max_budget_ratio": 0.00010092936587501424, "max_rel": 1.1102230246251565e-16, "max_ulp": 1, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 8, 17 ], "backend": "reference", "elements": 136, "expected_dtype": "float64", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "backend": "ort", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 2.220446049250313e-16, "max_budget_ratio": 0.00020185873175002848, "max_rel": 2.220446049250313e-16, "max_ulp": 2, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 1.3877787807814457e-17, "max_budget_ratio": 8.596324118225444e-05, "max_rel": 2.2588049270336617e-16, "max_ulp": 2, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 8, 17 ], "backend": "ort", "elements": 136, "expected_dtype": "float64", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 2.7755575615628914e-17, "max_budget_ratio": 0.0001650553953654569, "max_rel": 4.072171390944368e-16, "max_ulp": 3, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } } ], "comparison": { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 2.7755575615628914e-17, "max_budget_ratio": 0.00016505539536545687, "max_rel": 4.0721713909443665e-16, "max_ulp": 3, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, "comparison_key": "a543ff7ec1e8536bfeb5928c222ef6d5fcdff405250c88f2a41f94dfdaf3d11f", "contract": "probability", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "float64", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "Softmax/large_offset/float64/4x17/seed-20260906", "observation_id": "83387ced5d1c7ec2241f7816fb9ea04a0d06454398c484222434516d963144c7", "operation": "Softmax", "operators": [ "Softmax" ], "opset": 19, "optimization": "disabled", "output_hash": "916886b290c95e6d9c2b3f510f548b3857d6e04554007caacab78ec928f83a79", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.00325, "p95_ms": 0.0035042, "repeats": 7, "samples_ms": [ 0.00325, 0.003542, 0.003416, 0.003042, 0.00325, 0.003, 0.002959 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.007541, "p95_ms": 0.008787499999999998, "repeats": 7, "samples_ms": [ 0.009125, 0.008, 0.007541, 0.007459, 0.007458, 0.007667, 0.007458 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-13, "rtol": 1e-12 }, "variant": "large_offset", "warnings": [] }, { "artifacts": { "bundle": "observations/3b37d584c5419014d88c64ec7c307806feece990e62a83667a581e18ce4d2a62/reproducer.zip", "bundle_sha256": "1a0401dbf9ea806606c7911b0d8c0199b1433d50b5f8ff4c9693baa3a1e4d5bb", "case": "artifacts/2756cf12ec4e1fd845c3f6a6b56a6e2728659db4de7627f10ad8e98125f5ec14/case.json", "inputs": "artifacts/2756cf12ec4e1fd845c3f6a6b56a6e2728659db4de7627f10ad8e98125f5ec14/inputs.npz", "manifest": "artifacts/2756cf12ec4e1fd845c3f6a6b56a6e2728659db4de7627f10ad8e98125f5ec14/manifest.json", "model": "artifacts/2756cf12ec4e1fd845c3f6a6b56a6e2728659db4de7627f10ad8e98125f5ec14/model.onnx", "outputs": "observations/3b37d584c5419014d88c64ec7c307806feece990e62a83667a581e18ce4d2a62/outputs.npz" }, "candidate_group": "80a1b7c348350c723e225f319518fd9d17de20bcf89a95a77ebef080789f1972", "case_id": "2756cf12ec4e1fd845c3f6a6b56a6e2728659db4de7627f10ad8e98125f5ec14", "category": "softmax", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "backend": "reference", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 0.000362396240234375, "max_budget_ratio": 0.06589022549715909, "max_rel": 0.000362396240234375, "max_ulp": 1632087572480, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 8, 17 ], "backend": "reference", "elements": 136, "expected_dtype": "float16", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "backend": "ort", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 5.340576171875e-05, "max_budget_ratio": 0.009710138494318182, "max_rel": 5.340576171875e-05, "max_ulp": 481036337152, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 8, 17 ], "backend": "ort", "elements": 136, "expected_dtype": "float16", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } } ], "comparison": { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0001220703125, "max_budget_ratio": 0.10388531061707873, "max_rel": 0.0013253810470510272, "max_ulp": 2, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, "comparison_key": "75bdfae595d58a9b32498ff7c2e3d475d4fee4cff33f4eefe3a41f8573853466", "contract": "probability", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "float16", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "Softmax/ordinary/float16/4x17/seed-20260906", "observation_id": "3b37d584c5419014d88c64ec7c307806feece990e62a83667a581e18ce4d2a62", "operation": "Softmax", "operators": [ "Softmax" ], "opset": 19, "optimization": "all", "output_hash": "d05119b7965b981338767ce7756c4db5efa53de82a708a82b3ebe1b23201f3f2", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.005041, "p95_ms": 0.0057661000000000006, "repeats": 7, "samples_ms": [ 0.005791, 0.005708, 0.005084, 0.004958, 0.004917, 0.004958, 0.005041 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.009292, "p95_ms": 0.0102838, "repeats": 7, "samples_ms": [ 0.010459, 0.009875, 0.009167, 0.009292, 0.008917, 0.0095, 0.009042 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 0.0005, "rtol": 0.005 }, "variant": "ordinary", "warnings": [] }, { "artifacts": { "bundle": "observations/89afbda3dd59b443e997a35d07d78576067d6d024339f61cd4481ff3faf25cf2/reproducer.zip", "bundle_sha256": "f8d9c1cdf101e797562edcebb998c759dcc753f3f369254e04703f1cb96c4ccb", "case": "artifacts/2756cf12ec4e1fd845c3f6a6b56a6e2728659db4de7627f10ad8e98125f5ec14/case.json", "inputs": "artifacts/2756cf12ec4e1fd845c3f6a6b56a6e2728659db4de7627f10ad8e98125f5ec14/inputs.npz", "manifest": "artifacts/2756cf12ec4e1fd845c3f6a6b56a6e2728659db4de7627f10ad8e98125f5ec14/manifest.json", "model": "artifacts/2756cf12ec4e1fd845c3f6a6b56a6e2728659db4de7627f10ad8e98125f5ec14/model.onnx", "outputs": "observations/89afbda3dd59b443e997a35d07d78576067d6d024339f61cd4481ff3faf25cf2/outputs.npz" }, "candidate_group": "80a1b7c348350c723e225f319518fd9d17de20bcf89a95a77ebef080789f1972", "case_id": "2756cf12ec4e1fd845c3f6a6b56a6e2728659db4de7627f10ad8e98125f5ec14", "category": "softmax", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "backend": "reference", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 0.000362396240234375, "max_budget_ratio": 0.06589022549715909, "max_rel": 0.000362396240234375, "max_ulp": 1632087572480, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 8, 17 ], "backend": "reference", "elements": 136, "expected_dtype": "float16", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "backend": "ort", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 5.340576171875e-05, "max_budget_ratio": 0.009710138494318182, "max_rel": 5.340576171875e-05, "max_ulp": 481036337152, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 8, 17 ], "backend": "ort", "elements": 136, "expected_dtype": "float16", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } } ], "comparison": { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0001220703125, "max_budget_ratio": 0.10388531061707873, "max_rel": 0.0013253810470510272, "max_ulp": 2, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, "comparison_key": "4479224129364cd0cc01502095d525b98e9dd9d5c18ae20377b4c72a1433cd83", "contract": "probability", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "float16", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "Softmax/ordinary/float16/4x17/seed-20260906", "observation_id": "89afbda3dd59b443e997a35d07d78576067d6d024339f61cd4481ff3faf25cf2", "operation": "Softmax", "operators": [ "Softmax" ], "opset": 19, "optimization": "disabled", "output_hash": "d05119b7965b981338767ce7756c4db5efa53de82a708a82b3ebe1b23201f3f2", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.004333, "p95_ms": 0.0059749, "repeats": 7, "samples_ms": [ 0.00625, 0.005333, 0.004792, 0.004333, 0.004042, 0.003917, 0.004083 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.009542, "p95_ms": 0.012053699999999999, "repeats": 7, "samples_ms": [ 0.012666, 0.010625, 0.010625, 0.009542, 0.009291, 0.009041, 0.009125 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 0.0005, "rtol": 0.005 }, "variant": "ordinary", "warnings": [] }, { "artifacts": { "bundle": "observations/3de9d62a96b1608006ff34f585682ba404ec45c32dfa8c1902db6e8a18f06652/reproducer.zip", "bundle_sha256": "40910f1eac721d2e47db8014bbb63ec5ee89c69b99493a5403227839bfa03721", "case": "artifacts/7de56fdb0a0df3023a7787415ef2a8869973b08560ef1bb03b0774fcafc3fa66/case.json", "inputs": "artifacts/7de56fdb0a0df3023a7787415ef2a8869973b08560ef1bb03b0774fcafc3fa66/inputs.npz", "manifest": "artifacts/7de56fdb0a0df3023a7787415ef2a8869973b08560ef1bb03b0774fcafc3fa66/manifest.json", "model": "artifacts/7de56fdb0a0df3023a7787415ef2a8869973b08560ef1bb03b0774fcafc3fa66/model.onnx", "outputs": "observations/3de9d62a96b1608006ff34f585682ba404ec45c32dfa8c1902db6e8a18f06652/outputs.npz" }, "candidate_group": "f2ba02f8dbc266960ed236a23c8988b0b1f85ba86040188ba4e4eabc188ec71c", "case_id": "7de56fdb0a0df3023a7787415ef2a8869973b08560ef1bb03b0774fcafc3fa66", "category": "softmax", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "backend": "reference", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 5.4016709327697754e-08, "max_budget_ratio": 0.004910609938881614, "max_rel": 5.4016709327697754e-08, "max_ulp": 243269632, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 17 ], "backend": "reference", "elements": 136, "expected_dtype": "float32", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "backend": "ort", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 8.707866072654724e-08, "max_budget_ratio": 0.007916241884231566, "max_rel": 8.707866072654724e-08, "max_ulp": 503316480, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 17 ], "backend": "ort", "elements": 136, "expected_dtype": "float32", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 1.4901161193847656e-08, "max_budget_ratio": 0.007446247641838526, "max_rel": 1.7656915017356307e-07, "max_ulp": 2, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } } ], "comparison": { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 2.2351741790771484e-08, "max_budget_ratio": 0.011169370215209341, "max_rel": 2.2325754186302168e-07, "max_ulp": 3, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, "comparison_key": "aea52aa769d4b815081d385e833fac1fbc95f5b9cbac1749a9d768fb84dad12d", "contract": "probability", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "float32", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "Softmax/ordinary/float32/4x17/seed-20260906", "observation_id": "3de9d62a96b1608006ff34f585682ba404ec45c32dfa8c1902db6e8a18f06652", "operation": "Softmax", "operators": [ "Softmax" ], "opset": 19, "optimization": "all", "output_hash": "4cf5375f986a7947b477d16c1a63e5c92d582827267f16021cf45acf640c8954", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.002834, "p95_ms": 0.0030042, "repeats": 7, "samples_ms": [ 0.003042, 0.002916, 0.002834, 0.002875, 0.0025, 0.002625, 0.002416 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.008125, "p95_ms": 0.008712500000000001, "repeats": 7, "samples_ms": [ 0.008625, 0.008125, 0.00775, 0.007583, 0.007834, 0.00875, 0.008208 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-06, "rtol": 1e-05 }, "variant": "ordinary", "warnings": [] }, { "artifacts": { "bundle": "observations/7d1c947aeef5116525a250d230a173595d24669e96a6a25efd2a945c26650adc/reproducer.zip", "bundle_sha256": "6d527e9524271414e3a0d79b7893002e993ceef36be992c0b6dd2949ac5e3c48", "case": "artifacts/7de56fdb0a0df3023a7787415ef2a8869973b08560ef1bb03b0774fcafc3fa66/case.json", "inputs": "artifacts/7de56fdb0a0df3023a7787415ef2a8869973b08560ef1bb03b0774fcafc3fa66/inputs.npz", "manifest": "artifacts/7de56fdb0a0df3023a7787415ef2a8869973b08560ef1bb03b0774fcafc3fa66/manifest.json", "model": "artifacts/7de56fdb0a0df3023a7787415ef2a8869973b08560ef1bb03b0774fcafc3fa66/model.onnx", "outputs": "observations/7d1c947aeef5116525a250d230a173595d24669e96a6a25efd2a945c26650adc/outputs.npz" }, "candidate_group": "f2ba02f8dbc266960ed236a23c8988b0b1f85ba86040188ba4e4eabc188ec71c", "case_id": "7de56fdb0a0df3023a7787415ef2a8869973b08560ef1bb03b0774fcafc3fa66", "category": "softmax", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "backend": "reference", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 5.4016709327697754e-08, "max_budget_ratio": 0.004910609938881614, "max_rel": 5.4016709327697754e-08, "max_ulp": 243269632, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 17 ], "backend": "reference", "elements": 136, "expected_dtype": "float32", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "backend": "ort", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 8.707866072654724e-08, "max_budget_ratio": 0.007916241884231566, "max_rel": 8.707866072654724e-08, "max_ulp": 503316480, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 17 ], "backend": "ort", "elements": 136, "expected_dtype": "float32", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 1.4901161193847656e-08, "max_budget_ratio": 0.007446247641838526, "max_rel": 1.7656915017356307e-07, "max_ulp": 2, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } } ], "comparison": { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 2.2351741790771484e-08, "max_budget_ratio": 0.011169370215209341, "max_rel": 2.2325754186302168e-07, "max_ulp": 3, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, "comparison_key": "a06449fcc1f53c12ac2f99de8f4baf7fe13708a13defbe94bb732929f0233ca3", "contract": "probability", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "float32", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "Softmax/ordinary/float32/4x17/seed-20260906", "observation_id": "7d1c947aeef5116525a250d230a173595d24669e96a6a25efd2a945c26650adc", "operation": "Softmax", "operators": [ "Softmax" ], "opset": 19, "optimization": "disabled", "output_hash": "4cf5375f986a7947b477d16c1a63e5c92d582827267f16021cf45acf640c8954", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.003, "p95_ms": 0.0034376, "repeats": 7, "samples_ms": [ 0.003, 0.0035, 0.003292, 0.003041, 0.002791, 0.002917, 0.002709 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.008208, "p95_ms": 0.009195599999999998, "repeats": 7, "samples_ms": [ 0.009333, 0.008875, 0.00825, 0.008208, 0.008, 0.008208, 0.007917 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-06, "rtol": 1e-05 }, "variant": "ordinary", "warnings": [] }, { "artifacts": { "bundle": "observations/b732b35a82b7803eff6c81d39ab73407db291eca9e51d4d2f2d99de61a14eea6/reproducer.zip", "bundle_sha256": "36cd3a4766e3e2c1082c3ef6a806fc15209248be010935f8cf7e599ca9b20737", "case": "artifacts/711945ce26d8f77be60b0314c15dd9567523082f0cc64ff88a791506f7b0b9ba/case.json", "inputs": "artifacts/711945ce26d8f77be60b0314c15dd9567523082f0cc64ff88a791506f7b0b9ba/inputs.npz", "manifest": "artifacts/711945ce26d8f77be60b0314c15dd9567523082f0cc64ff88a791506f7b0b9ba/manifest.json", "model": "artifacts/711945ce26d8f77be60b0314c15dd9567523082f0cc64ff88a791506f7b0b9ba/model.onnx", "outputs": "observations/b732b35a82b7803eff6c81d39ab73407db291eca9e51d4d2f2d99de61a14eea6/outputs.npz" }, "candidate_group": "3dc7126035bd2bc0b3b55ad71e2ad10711cdf3cecfce5b6e3cb9c6f53f168dfa", "case_id": "711945ce26d8f77be60b0314c15dd9567523082f0cc64ff88a791506f7b0b9ba", "category": "softmax", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "backend": "reference", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 2.220446049250313e-16, "max_budget_ratio": 0.00020185873175002848, "max_rel": 2.220446049250313e-16, "max_ulp": 2, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 8, 17 ], "backend": "reference", "elements": 136, "expected_dtype": "float64", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "backend": "ort", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 2.220446049250313e-16, "max_budget_ratio": 0.00020185873175002848, "max_rel": 2.220446049250313e-16, "max_ulp": 1, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 5.551115123125783e-17, "max_budget_ratio": 0.0001660448311794271, "max_rel": 3.4075213880595625e-16, "max_ulp": 2, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 8, 17 ], "backend": "ort", "elements": 136, "expected_dtype": "float64", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 5.551115123125783e-17, "max_budget_ratio": 0.0001660448311794271, "max_rel": 3.4075213880595625e-16, "max_ulp": 2, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } } ], "comparison": { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 5.551115123125783e-17, "max_budget_ratio": 0.0001660448311794271, "max_rel": 4.1145083158674427e-16, "max_ulp": 3, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, "comparison_key": "e6c391dc869550392c8550fe685983590cee78bf8225e8782482c9af2183a999", "contract": "probability", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "float64", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "Softmax/ordinary/float64/4x17/seed-20260906", "observation_id": "b732b35a82b7803eff6c81d39ab73407db291eca9e51d4d2f2d99de61a14eea6", "operation": "Softmax", "operators": [ "Softmax" ], "opset": 19, "optimization": "all", "output_hash": "ef9225f802a4460e7ffbd76da074365f797334287912492cc9cb3d33d0d8216f", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.003209, "p95_ms": 0.0035794, "repeats": 7, "samples_ms": [ 0.003375, 0.003667, 0.003291, 0.003209, 0.003125, 0.003042, 0.003125 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.008125, "p95_ms": 0.0101456, "repeats": 7, "samples_ms": [ 0.01, 0.010208, 0.008125, 0.008, 0.008, 0.008125, 0.007834 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-13, "rtol": 1e-12 }, "variant": "ordinary", "warnings": [] }, { "artifacts": { "bundle": "observations/191b2627c04312f94c501189e8c40fa0be8eff12cd50de15d00d4016cc856878/reproducer.zip", "bundle_sha256": "c9101e4793c0b06fe93afd3617325aeabce6bcebb0298e4f9bbefed7b2bc1726", "case": "artifacts/711945ce26d8f77be60b0314c15dd9567523082f0cc64ff88a791506f7b0b9ba/case.json", "inputs": "artifacts/711945ce26d8f77be60b0314c15dd9567523082f0cc64ff88a791506f7b0b9ba/inputs.npz", "manifest": "artifacts/711945ce26d8f77be60b0314c15dd9567523082f0cc64ff88a791506f7b0b9ba/manifest.json", "model": "artifacts/711945ce26d8f77be60b0314c15dd9567523082f0cc64ff88a791506f7b0b9ba/model.onnx", "outputs": "observations/191b2627c04312f94c501189e8c40fa0be8eff12cd50de15d00d4016cc856878/outputs.npz" }, "candidate_group": "3dc7126035bd2bc0b3b55ad71e2ad10711cdf3cecfce5b6e3cb9c6f53f168dfa", "case_id": "711945ce26d8f77be60b0314c15dd9567523082f0cc64ff88a791506f7b0b9ba", "category": "softmax", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "backend": "reference", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 2.220446049250313e-16, "max_budget_ratio": 0.00020185873175002848, "max_rel": 2.220446049250313e-16, "max_ulp": 2, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 8, 17 ], "backend": "reference", "elements": 136, "expected_dtype": "float64", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "backend": "ort", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 2.220446049250313e-16, "max_budget_ratio": 0.00020185873175002848, "max_rel": 2.220446049250313e-16, "max_ulp": 1, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 5.551115123125783e-17, "max_budget_ratio": 0.0001660448311794271, "max_rel": 3.4075213880595625e-16, "max_ulp": 2, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 8, 17 ], "backend": "ort", "elements": 136, "expected_dtype": "float64", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 5.551115123125783e-17, "max_budget_ratio": 0.0001660448311794271, "max_rel": 3.4075213880595625e-16, "max_ulp": 2, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } } ], "comparison": { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 5.551115123125783e-17, "max_budget_ratio": 0.0001660448311794271, "max_rel": 4.1145083158674427e-16, "max_ulp": 3, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, "comparison_key": "b7cbd3e8432acaf853f49ac07ec8e80b0d40f655ae7282ea1b930e6d7f10f789", "contract": "probability", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "float64", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "Softmax/ordinary/float64/4x17/seed-20260906", "observation_id": "191b2627c04312f94c501189e8c40fa0be8eff12cd50de15d00d4016cc856878", "operation": "Softmax", "operators": [ "Softmax" ], "opset": 19, "optimization": "disabled", "output_hash": "ef9225f802a4460e7ffbd76da074365f797334287912492cc9cb3d33d0d8216f", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.00325, "p95_ms": 0.0037874, "repeats": 7, "samples_ms": [ 0.003583, 0.003875, 0.003459, 0.003125, 0.003167, 0.00325, 0.003208 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.008416, "p95_ms": 0.0092625, "repeats": 7, "samples_ms": [ 0.009375, 0.009, 0.008625, 0.008334, 0.00825, 0.008416, 0.008041 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-13, "rtol": 1e-12 }, "variant": "ordinary", "warnings": [] }, { "artifacts": { "bundle": "observations/e973672bf3d1da8197da965d4feaf7a665d71034e87b5efc5fe7e6cffc41b766/reproducer.zip", "bundle_sha256": "d15dd7d0faa5ff475ee8b46496b56cbbb2011acd41542ad3808af687b81f1b00", "case": "artifacts/0ecc019230e13044fdf908f9ba564719980f7e2a76daccf5d4ac8ad6e3ea1e1a/case.json", "inputs": "artifacts/0ecc019230e13044fdf908f9ba564719980f7e2a76daccf5d4ac8ad6e3ea1e1a/inputs.npz", "manifest": "artifacts/0ecc019230e13044fdf908f9ba564719980f7e2a76daccf5d4ac8ad6e3ea1e1a/manifest.json", "model": "artifacts/0ecc019230e13044fdf908f9ba564719980f7e2a76daccf5d4ac8ad6e3ea1e1a/model.onnx", "outputs": "observations/e973672bf3d1da8197da965d4feaf7a665d71034e87b5efc5fe7e6cffc41b766/outputs.npz" }, "candidate_group": "969aeb10b2e3fcb04b1c65aadaa665bc88615b8f115e226e896572f3fb0f50e3", "case_id": "0ecc019230e13044fdf908f9ba564719980f7e2a76daccf5d4ac8ad6e3ea1e1a", "category": "softmax", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "backend": "reference", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 1.2014061212539673e-07, "max_budget_ratio": 0.01092187382958152, "max_rel": 1.2014061212539673e-07, "max_ulp": 541065216, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 79 ], "backend": "reference", "elements": 316, "expected_dtype": "float32", "expected_shape": [ 4, 79 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 79 ], "backend": "reference", "elements": 316, "expected_dtype": "float32", "expected_shape": [ 4, 79 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 79 ], "backend": "reference", "elements": 316, "expected_dtype": "float32", "expected_shape": [ 4, 79 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 79 ], "backend": "reference", "elements": 632, "expected_dtype": "float32", "expected_shape": [ 8, 79 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 79 ], "backend": "reference", "elements": 316, "expected_dtype": "float32", "expected_shape": [ 4, 79 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "backend": "ort", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 1.2759119272232056e-07, "max_budget_ratio": 0.011599199338392777, "max_rel": 1.2759119272232056e-07, "max_ulp": 574619648, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 79 ], "backend": "ort", "elements": 316, "expected_dtype": "float32", "expected_shape": [ 4, 79 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 79 ], "backend": "ort", "elements": 316, "expected_dtype": "float32", "expected_shape": [ 4, 79 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 79 ], "backend": "ort", "elements": 316, "expected_dtype": "float32", "expected_shape": [ 4, 79 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 79 ], "backend": "ort", "elements": 632, "expected_dtype": "float32", "expected_shape": [ 8, 79 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 79 ], "backend": "ort", "elements": 316, "expected_dtype": "float32", "expected_shape": [ 4, 79 ], "first_mismatch": null, "max_abs": 3.725290298461914e-09, "max_budget_ratio": 0.0031694757074837115, "max_rel": 2.1416734093684505e-07, "max_ulp": 2, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } } ], "comparison": { "actual_dtype": "float32", "actual_shape": [ 4, 79 ], "elements": 316, "expected_dtype": "float32", "expected_shape": [ 4, 79 ], "first_mismatch": null, "max_abs": 3.725290298461914e-09, "max_budget_ratio": 0.0032123276181881106, "max_rel": 2.3328895788539424e-07, "max_ulp": 3, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, "comparison_key": "59aac5e141674fe4c65391d3432b52d051ab1b0cafa350b33e30cdf796a2bfee", "contract": "probability", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "float32", "shape": [ 4, 79 ] } }, "ir_version": 10, "name": "Softmax/random-000/float32/4x79/seed-20260906", "observation_id": "e973672bf3d1da8197da965d4feaf7a665d71034e87b5efc5fe7e6cffc41b766", "operation": "Softmax", "operators": [ "Softmax" ], "opset": 19, "optimization": "all", "output_hash": "cc0201954ad2d18075804a8413329175d96ba01cbfa828bf1539a8c65ccc6009", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.003208, "p95_ms": 0.0044248, "repeats": 7, "samples_ms": [ 0.003, 0.003666, 0.003375, 0.003083, 0.00475, 0.003208, 0.003166 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.008875, "p95_ms": 0.0098, "repeats": 7, "samples_ms": [ 0.009625, 0.009875, 0.009, 0.008875, 0.008416, 0.008834, 0.008458 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-06, "rtol": 1e-05 }, "variant": "random-000", "warnings": [] }, { "artifacts": { "bundle": "observations/ee96f03e70600993f07234183efb370cefcb949f80cc6c5eb0467043a0fe8470/reproducer.zip", "bundle_sha256": "44b63def70c859d1be391c0858166b939d6ba5c8a3076e6038336e64ae51a8e1", "case": "artifacts/0ecc019230e13044fdf908f9ba564719980f7e2a76daccf5d4ac8ad6e3ea1e1a/case.json", "inputs": "artifacts/0ecc019230e13044fdf908f9ba564719980f7e2a76daccf5d4ac8ad6e3ea1e1a/inputs.npz", "manifest": "artifacts/0ecc019230e13044fdf908f9ba564719980f7e2a76daccf5d4ac8ad6e3ea1e1a/manifest.json", "model": "artifacts/0ecc019230e13044fdf908f9ba564719980f7e2a76daccf5d4ac8ad6e3ea1e1a/model.onnx", "outputs": "observations/ee96f03e70600993f07234183efb370cefcb949f80cc6c5eb0467043a0fe8470/outputs.npz" }, "candidate_group": "969aeb10b2e3fcb04b1c65aadaa665bc88615b8f115e226e896572f3fb0f50e3", "case_id": "0ecc019230e13044fdf908f9ba564719980f7e2a76daccf5d4ac8ad6e3ea1e1a", "category": "softmax", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "backend": "reference", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 1.2014061212539673e-07, "max_budget_ratio": 0.01092187382958152, "max_rel": 1.2014061212539673e-07, "max_ulp": 541065216, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 79 ], "backend": "reference", "elements": 316, "expected_dtype": "float32", "expected_shape": [ 4, 79 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 79 ], "backend": "reference", "elements": 316, "expected_dtype": "float32", "expected_shape": [ 4, 79 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 79 ], "backend": "reference", "elements": 316, "expected_dtype": "float32", "expected_shape": [ 4, 79 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 79 ], "backend": "reference", "elements": 632, "expected_dtype": "float32", "expected_shape": [ 8, 79 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 79 ], "backend": "reference", "elements": 316, "expected_dtype": "float32", "expected_shape": [ 4, 79 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "backend": "ort", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 1.2759119272232056e-07, "max_budget_ratio": 0.011599199338392777, "max_rel": 1.2759119272232056e-07, "max_ulp": 574619648, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 79 ], "backend": "ort", "elements": 316, "expected_dtype": "float32", "expected_shape": [ 4, 79 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 79 ], "backend": "ort", "elements": 316, "expected_dtype": "float32", "expected_shape": [ 4, 79 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 79 ], "backend": "ort", "elements": 316, "expected_dtype": "float32", "expected_shape": [ 4, 79 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 79 ], "backend": "ort", "elements": 632, "expected_dtype": "float32", "expected_shape": [ 8, 79 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 79 ], "backend": "ort", "elements": 316, "expected_dtype": "float32", "expected_shape": [ 4, 79 ], "first_mismatch": null, "max_abs": 3.725290298461914e-09, "max_budget_ratio": 0.0031694757074837115, "max_rel": 2.1416734093684505e-07, "max_ulp": 2, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } } ], "comparison": { "actual_dtype": "float32", "actual_shape": [ 4, 79 ], "elements": 316, "expected_dtype": "float32", "expected_shape": [ 4, 79 ], "first_mismatch": null, "max_abs": 3.725290298461914e-09, "max_budget_ratio": 0.0032123276181881106, "max_rel": 2.3328895788539424e-07, "max_ulp": 3, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, "comparison_key": "196eb69bc04a135969a0e01f3d7d3982028f61e6bd8a0c1b6b921003e8a7d990", "contract": "probability", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "float32", "shape": [ 4, 79 ] } }, "ir_version": 10, "name": "Softmax/random-000/float32/4x79/seed-20260906", "observation_id": "ee96f03e70600993f07234183efb370cefcb949f80cc6c5eb0467043a0fe8470", "operation": "Softmax", "operators": [ "Softmax" ], "opset": 19, "optimization": "disabled", "output_hash": "cc0201954ad2d18075804a8413329175d96ba01cbfa828bf1539a8c65ccc6009", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.003, "p95_ms": 0.0035957999999999997, "repeats": 7, "samples_ms": [ 0.003292, 0.003708, 0.003334, 0.003, 0.002709, 0.00275, 0.00275 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.008667, "p95_ms": 0.0096581, "repeats": 7, "samples_ms": [ 0.009833, 0.00925, 0.008834, 0.008625, 0.008667, 0.008417, 0.008292 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-06, "rtol": 1e-05 }, "variant": "random-000", "warnings": [] }, { "artifacts": { "bundle": "observations/440a01dcee7f116f61c8046e4c32a18234d8c443c12f642e64058b6dc915bf87/reproducer.zip", "bundle_sha256": "b8c06c6bd98abf3275d9199febda5331d465c32a087ad26fe7728cca08de94ff", "case": "artifacts/a2ff996dbbc4d2e883ec9badfbed14acbe166db95a7201ad46bd3a7269a48aae/case.json", "inputs": "artifacts/a2ff996dbbc4d2e883ec9badfbed14acbe166db95a7201ad46bd3a7269a48aae/inputs.npz", "manifest": "artifacts/a2ff996dbbc4d2e883ec9badfbed14acbe166db95a7201ad46bd3a7269a48aae/manifest.json", "model": "artifacts/a2ff996dbbc4d2e883ec9badfbed14acbe166db95a7201ad46bd3a7269a48aae/model.onnx", "outputs": "observations/440a01dcee7f116f61c8046e4c32a18234d8c443c12f642e64058b6dc915bf87/outputs.npz" }, "candidate_group": "fb50f28f10087218ec0053576dab04221266ceebe70140e38dfdfaaf5155fe6b", "case_id": "a2ff996dbbc4d2e883ec9badfbed14acbe166db95a7201ad46bd3a7269a48aae", "category": "softmax", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "backend": "reference", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 1.202854507109663e-07, "max_budget_ratio": 0.010935040973724207, "max_rel": 1.202854507109663e-07, "max_ulp": 1083435022, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 71 ], "backend": "reference", "elements": 284, "expected_dtype": "float32", "expected_shape": [ 4, 71 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 71 ], "backend": "reference", "elements": 284, "expected_dtype": "float32", "expected_shape": [ 4, 71 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 71 ], "backend": "reference", "elements": 284, "expected_dtype": "float32", "expected_shape": [ 4, 71 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 71 ], "backend": "reference", "elements": 568, "expected_dtype": "float32", "expected_shape": [ 8, 71 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 71 ], "backend": "reference", "elements": 284, "expected_dtype": "float32", "expected_shape": [ 4, 71 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "backend": "ort", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 1.970070857559847e-07, "max_budget_ratio": 0.01790973506872588, "max_rel": 1.970070857559847e-07, "max_ulp": 1774482076, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 71 ], "backend": "ort", "elements": 284, "expected_dtype": "float32", "expected_shape": [ 4, 71 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 71 ], "backend": "ort", "elements": 284, "expected_dtype": "float32", "expected_shape": [ 4, 71 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 71 ], "backend": "ort", "elements": 284, "expected_dtype": "float32", "expected_shape": [ 4, 71 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 71 ], "backend": "ort", "elements": 568, "expected_dtype": "float32", "expected_shape": [ 8, 71 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 71 ], "backend": "ort", "elements": 284, "expected_dtype": "float32", "expected_shape": [ 4, 71 ], "first_mismatch": null, "max_abs": 1.7881393432617188e-07, "max_budget_ratio": 0.017966179816331262, "max_rel": 2.699055573464289e-07, "max_ulp": 4, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } } ], "comparison": { "actual_dtype": "float32", "actual_shape": [ 4, 71 ], "elements": 284, "expected_dtype": "float32", "expected_shape": [ 4, 71 ], "first_mismatch": null, "max_abs": 1.1920928955078125e-07, "max_budget_ratio": 0.01166642544387657, "max_rel": 2.429169860350264e-07, "max_ulp": 3, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, "comparison_key": "f9ca6364ef185dec1913912628f11f11841ad66c7d70e33f2ecaacb541cf7571", "contract": "probability", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "float32", "shape": [ 4, 71 ] } }, "ir_version": 10, "name": "Softmax/random-007/float32/4x71/seed-20260906", "observation_id": "440a01dcee7f116f61c8046e4c32a18234d8c443c12f642e64058b6dc915bf87", "operation": "Softmax", "operators": [ "Softmax" ], "opset": 19, "optimization": "all", "output_hash": "f417e6b20ea6b29a8a2635b019ae378a7a01d913f182058c1ba17d6cc47a2d8e", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.003042, "p95_ms": 0.003417, "repeats": 7, "samples_ms": [ 0.003041, 0.003417, 0.003167, 0.003417, 0.00275, 0.00275, 0.003042 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.008791, "p95_ms": 0.0094036, "repeats": 7, "samples_ms": [ 0.009541, 0.009083, 0.00875, 0.008709, 0.008791, 0.008959, 0.0085 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-06, "rtol": 1e-05 }, "variant": "random-007", "warnings": [] }, { "artifacts": { "bundle": "observations/f00b737b37268767915db31727c97c385da032752ced97dd1ea48fb429e5730e/reproducer.zip", "bundle_sha256": "9c6afaf2aade1d3da756ca0e6c6a669ce62c72bf092e1f523996758cf4f85120", "case": "artifacts/a2ff996dbbc4d2e883ec9badfbed14acbe166db95a7201ad46bd3a7269a48aae/case.json", "inputs": "artifacts/a2ff996dbbc4d2e883ec9badfbed14acbe166db95a7201ad46bd3a7269a48aae/inputs.npz", "manifest": "artifacts/a2ff996dbbc4d2e883ec9badfbed14acbe166db95a7201ad46bd3a7269a48aae/manifest.json", "model": "artifacts/a2ff996dbbc4d2e883ec9badfbed14acbe166db95a7201ad46bd3a7269a48aae/model.onnx", "outputs": "observations/f00b737b37268767915db31727c97c385da032752ced97dd1ea48fb429e5730e/outputs.npz" }, "candidate_group": "fb50f28f10087218ec0053576dab04221266ceebe70140e38dfdfaaf5155fe6b", "case_id": "a2ff996dbbc4d2e883ec9badfbed14acbe166db95a7201ad46bd3a7269a48aae", "category": "softmax", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "backend": "reference", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 1.202854507109663e-07, "max_budget_ratio": 0.010935040973724207, "max_rel": 1.202854507109663e-07, "max_ulp": 1083435022, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 71 ], "backend": "reference", "elements": 284, "expected_dtype": "float32", "expected_shape": [ 4, 71 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 71 ], "backend": "reference", "elements": 284, "expected_dtype": "float32", "expected_shape": [ 4, 71 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 71 ], "backend": "reference", "elements": 284, "expected_dtype": "float32", "expected_shape": [ 4, 71 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 71 ], "backend": "reference", "elements": 568, "expected_dtype": "float32", "expected_shape": [ 8, 71 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 71 ], "backend": "reference", "elements": 284, "expected_dtype": "float32", "expected_shape": [ 4, 71 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "backend": "ort", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 1.970070857559847e-07, "max_budget_ratio": 0.01790973506872588, "max_rel": 1.970070857559847e-07, "max_ulp": 1774482076, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 71 ], "backend": "ort", "elements": 284, "expected_dtype": "float32", "expected_shape": [ 4, 71 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 71 ], "backend": "ort", "elements": 284, "expected_dtype": "float32", "expected_shape": [ 4, 71 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 71 ], "backend": "ort", "elements": 284, "expected_dtype": "float32", "expected_shape": [ 4, 71 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 71 ], "backend": "ort", "elements": 568, "expected_dtype": "float32", "expected_shape": [ 8, 71 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 71 ], "backend": "ort", "elements": 284, "expected_dtype": "float32", "expected_shape": [ 4, 71 ], "first_mismatch": null, "max_abs": 1.7881393432617188e-07, "max_budget_ratio": 0.017966179816331262, "max_rel": 2.699055573464289e-07, "max_ulp": 4, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } } ], "comparison": { "actual_dtype": "float32", "actual_shape": [ 4, 71 ], "elements": 284, "expected_dtype": "float32", "expected_shape": [ 4, 71 ], "first_mismatch": null, "max_abs": 1.1920928955078125e-07, "max_budget_ratio": 0.01166642544387657, "max_rel": 2.429169860350264e-07, "max_ulp": 3, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, "comparison_key": "ad53f5ba728df7459764f85de93f7dc224da81c015575cbff0dd41fdd1fba843", "contract": "probability", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "float32", "shape": [ 4, 71 ] } }, "ir_version": 10, "name": "Softmax/random-007/float32/4x71/seed-20260906", "observation_id": "f00b737b37268767915db31727c97c385da032752ced97dd1ea48fb429e5730e", "operation": "Softmax", "operators": [ "Softmax" ], "opset": 19, "optimization": "disabled", "output_hash": "f417e6b20ea6b29a8a2635b019ae378a7a01d913f182058c1ba17d6cc47a2d8e", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.003333, "p95_ms": 0.0035660999999999996, "repeats": 7, "samples_ms": [ 0.003333, 0.003666, 0.003333, 0.003333, 0.003167, 0.002917, 0.003083 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.008833, "p95_ms": 0.0096749, "repeats": 7, "samples_ms": [ 0.009875, 0.009208, 0.008833, 0.008833, 0.008667, 0.0085, 0.008458 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-06, "rtol": 1e-05 }, "variant": "random-007", "warnings": [] }, { "artifacts": { "bundle": "observations/1cb263a03b7a836abd16f33adc0583e8fc0a6df670481dcd44384b78e8f16b00/reproducer.zip", "bundle_sha256": "4d2f0f48a2f67d33a0f8992381760770b3a39d35652b66459128481ad4c91cd4", "case": "artifacts/abddaa2096db41ad07fccdb4e031f1577a1f12a112150ecafee3ee3a9078e13b/case.json", "inputs": "artifacts/abddaa2096db41ad07fccdb4e031f1577a1f12a112150ecafee3ee3a9078e13b/inputs.npz", "manifest": "artifacts/abddaa2096db41ad07fccdb4e031f1577a1f12a112150ecafee3ee3a9078e13b/manifest.json", "model": "artifacts/abddaa2096db41ad07fccdb4e031f1577a1f12a112150ecafee3ee3a9078e13b/model.onnx", "outputs": "observations/1cb263a03b7a836abd16f33adc0583e8fc0a6df670481dcd44384b78e8f16b00/outputs.npz" }, "candidate_group": "7aa0eff85431dcdfa190c4970c57eac0f60f1de3d04ea8d1690e455df37c5be7", "case_id": "abddaa2096db41ad07fccdb4e031f1577a1f12a112150ecafee3ee3a9078e13b", "category": "softmax", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "backend": "reference", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 7.479376484820222e-08, "max_budget_ratio": 0.006799433168018383, "max_rel": 7.479376484820222e-08, "max_ulp": 673682343, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 101 ], "backend": "reference", "elements": 404, "expected_dtype": "float32", "expected_shape": [ 4, 101 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 101 ], "backend": "reference", "elements": 404, "expected_dtype": "float32", "expected_shape": [ 4, 101 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 101 ], "backend": "reference", "elements": 404, "expected_dtype": "float32", "expected_shape": [ 4, 101 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 101 ], "backend": "reference", "elements": 808, "expected_dtype": "float32", "expected_shape": [ 8, 101 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 101 ], "backend": "reference", "elements": 404, "expected_dtype": "float32", "expected_shape": [ 4, 101 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "backend": "ort", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 9.099816145052841e-08, "max_budget_ratio": 0.008272560131866218, "max_rel": 9.099816145052841e-08, "max_ulp": 819638572, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 101 ], "backend": "ort", "elements": 404, "expected_dtype": "float32", "expected_shape": [ 4, 101 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 101 ], "backend": "ort", "elements": 404, "expected_dtype": "float32", "expected_shape": [ 4, 101 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 101 ], "backend": "ort", "elements": 404, "expected_dtype": "float32", "expected_shape": [ 4, 101 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 101 ], "backend": "ort", "elements": 808, "expected_dtype": "float32", "expected_shape": [ 8, 101 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 101 ], "backend": "ort", "elements": 404, "expected_dtype": "float32", "expected_shape": [ 4, 101 ], "first_mismatch": null, "max_abs": 2.9802322387695312e-08, "max_budget_ratio": 0.00833003588862682, "max_rel": 2.5457088931989146e+269, "max_ulp": 4042244, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } } ], "comparison": { "actual_dtype": "float32", "actual_shape": [ 4, 101 ], "elements": 404, "expected_dtype": "float32", "expected_shape": [ 4, 101 ], "first_mismatch": null, "max_abs": 1.1920928955078125e-07, "max_budget_ratio": 0.013098583832200538, "max_rel": 1.0, "max_ulp": 4042244, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, "comparison_key": "f449dc5231d1f62e6bd2ec166e2e54eecbe1ea7a21647bd2519419333880e0dc", "contract": "probability", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "float32", "shape": [ 4, 101 ] } }, "ir_version": 10, "name": "Softmax/random-014/float32/4x101/seed-20260906", "observation_id": "1cb263a03b7a836abd16f33adc0583e8fc0a6df670481dcd44384b78e8f16b00", "operation": "Softmax", "operators": [ "Softmax" ], "opset": 19, "optimization": "all", "output_hash": "127cf79bfb5bc9ef5ccba5bbd9c4540b781cc234d036fb45d6178fa11898970d", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.003083, "p95_ms": 0.0034714999999999998, "repeats": 7, "samples_ms": [ 0.003209, 0.003584, 0.003125, 0.003041, 0.002959, 0.002958, 0.003083 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.010459, "p95_ms": 0.0121125, "repeats": 7, "samples_ms": [ 0.012375, 0.0115, 0.01075, 0.010458, 0.010167, 0.010459, 0.010208 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-06, "rtol": 1e-05 }, "variant": "random-014", "warnings": [ { "backend": "reference", "message": "underflow encountered in divide" }, { "backend": "reference", "message": "underflow encountered in exp" } ] }, { "artifacts": { "bundle": "observations/a802a3f361fa071fd506e6f418d73654d13d429ffe534549d1492ab4ae504677/reproducer.zip", "bundle_sha256": "07405acea423402b02c848d8b6a7d233c8b128596537f041d3b237c9a66c9d97", "case": "artifacts/abddaa2096db41ad07fccdb4e031f1577a1f12a112150ecafee3ee3a9078e13b/case.json", "inputs": "artifacts/abddaa2096db41ad07fccdb4e031f1577a1f12a112150ecafee3ee3a9078e13b/inputs.npz", "manifest": "artifacts/abddaa2096db41ad07fccdb4e031f1577a1f12a112150ecafee3ee3a9078e13b/manifest.json", "model": "artifacts/abddaa2096db41ad07fccdb4e031f1577a1f12a112150ecafee3ee3a9078e13b/model.onnx", "outputs": "observations/a802a3f361fa071fd506e6f418d73654d13d429ffe534549d1492ab4ae504677/outputs.npz" }, "candidate_group": "7aa0eff85431dcdfa190c4970c57eac0f60f1de3d04ea8d1690e455df37c5be7", "case_id": "abddaa2096db41ad07fccdb4e031f1577a1f12a112150ecafee3ee3a9078e13b", "category": "softmax", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "backend": "reference", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 7.479376484820222e-08, "max_budget_ratio": 0.006799433168018383, "max_rel": 7.479376484820222e-08, "max_ulp": 673682343, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 101 ], "backend": "reference", "elements": 404, "expected_dtype": "float32", "expected_shape": [ 4, 101 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 101 ], "backend": "reference", "elements": 404, "expected_dtype": "float32", "expected_shape": [ 4, 101 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 101 ], "backend": "reference", "elements": 404, "expected_dtype": "float32", "expected_shape": [ 4, 101 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 101 ], "backend": "reference", "elements": 808, "expected_dtype": "float32", "expected_shape": [ 8, 101 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 101 ], "backend": "reference", "elements": 404, "expected_dtype": "float32", "expected_shape": [ 4, 101 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "backend": "ort", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 9.099816145052841e-08, "max_budget_ratio": 0.008272560131866218, "max_rel": 9.099816145052841e-08, "max_ulp": 819638572, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 101 ], "backend": "ort", "elements": 404, "expected_dtype": "float32", "expected_shape": [ 4, 101 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 101 ], "backend": "ort", "elements": 404, "expected_dtype": "float32", "expected_shape": [ 4, 101 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 101 ], "backend": "ort", "elements": 404, "expected_dtype": "float32", "expected_shape": [ 4, 101 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 101 ], "backend": "ort", "elements": 808, "expected_dtype": "float32", "expected_shape": [ 8, 101 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 101 ], "backend": "ort", "elements": 404, "expected_dtype": "float32", "expected_shape": [ 4, 101 ], "first_mismatch": null, "max_abs": 2.9802322387695312e-08, "max_budget_ratio": 0.00833003588862682, "max_rel": 2.5457088931989146e+269, "max_ulp": 4042244, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } } ], "comparison": { "actual_dtype": "float32", "actual_shape": [ 4, 101 ], "elements": 404, "expected_dtype": "float32", "expected_shape": [ 4, 101 ], "first_mismatch": null, "max_abs": 1.1920928955078125e-07, "max_budget_ratio": 0.013098583832200538, "max_rel": 1.0, "max_ulp": 4042244, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, "comparison_key": "c6e06febc8d4198584a93edf0852f7f5e6040d7cad1b48e10e9a284d9d24682c", "contract": "probability", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "float32", "shape": [ 4, 101 ] } }, "ir_version": 10, "name": "Softmax/random-014/float32/4x101/seed-20260906", "observation_id": "a802a3f361fa071fd506e6f418d73654d13d429ffe534549d1492ab4ae504677", "operation": "Softmax", "operators": [ "Softmax" ], "opset": 19, "optimization": "disabled", "output_hash": "127cf79bfb5bc9ef5ccba5bbd9c4540b781cc234d036fb45d6178fa11898970d", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.003083, "p95_ms": 0.0035418999999999997, "repeats": 7, "samples_ms": [ 0.00325, 0.003667, 0.003042, 0.003083, 0.002834, 0.003125, 0.00275 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.009791, "p95_ms": 0.011362699999999998, "repeats": 7, "samples_ms": [ 0.01175, 0.010459, 0.010041, 0.009791, 0.009458, 0.009209, 0.009125 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-06, "rtol": 1e-05 }, "variant": "random-014", "warnings": [ { "backend": "reference", "message": "underflow encountered in divide" }, { "backend": "reference", "message": "underflow encountered in exp" } ] }, { "artifacts": { "bundle": "observations/fe7223396ffcd2e8221e1906303d75983e02aa864b50cc2d83eb8c10ed155a57/reproducer.zip", "bundle_sha256": "29aef7a4f61efc1d02e1ec90cee6db9d5efe7a327a2ca006c1a53053671274ee", "case": "artifacts/6e739d54f1eae43e8852f197450cf3c27d6a0ed62fab0cc7526e0699c3c9324c/case.json", "inputs": "artifacts/6e739d54f1eae43e8852f197450cf3c27d6a0ed62fab0cc7526e0699c3c9324c/inputs.npz", "manifest": "artifacts/6e739d54f1eae43e8852f197450cf3c27d6a0ed62fab0cc7526e0699c3c9324c/manifest.json", "model": "artifacts/6e739d54f1eae43e8852f197450cf3c27d6a0ed62fab0cc7526e0699c3c9324c/model.onnx", "outputs": "observations/fe7223396ffcd2e8221e1906303d75983e02aa864b50cc2d83eb8c10ed155a57/outputs.npz" }, "candidate_group": "aec977c93f96d9177a5b9570d1a9a8f34a5f159f8959977d62a2091818bf0db4", "case_id": "6e739d54f1eae43e8852f197450cf3c27d6a0ed62fab0cc7526e0699c3c9324c", "category": "softmax", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "backend": "reference", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 7.194103135432783e-08, "max_budget_ratio": 0.006540093759484347, "max_rel": 7.194103135432783e-08, "max_ulp": 489357312, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 85 ], "backend": "reference", "elements": 340, "expected_dtype": "float32", "expected_shape": [ 4, 85 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 85 ], "backend": "reference", "elements": 340, "expected_dtype": "float32", "expected_shape": [ 4, 85 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 85 ], "backend": "reference", "elements": 340, "expected_dtype": "float32", "expected_shape": [ 4, 85 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 85 ], "backend": "reference", "elements": 680, "expected_dtype": "float32", "expected_shape": [ 8, 85 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 85 ], "backend": "reference", "elements": 340, "expected_dtype": "float32", "expected_shape": [ 4, 85 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "backend": "ort", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 7.194103135432783e-08, "max_budget_ratio": 0.006540093759484347, "max_rel": 7.194103135432783e-08, "max_ulp": 489357312, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 85 ], "backend": "ort", "elements": 340, "expected_dtype": "float32", "expected_shape": [ 4, 85 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 85 ], "backend": "ort", "elements": 340, "expected_dtype": "float32", "expected_shape": [ 4, 85 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 85 ], "backend": "ort", "elements": 340, "expected_dtype": "float32", "expected_shape": [ 4, 85 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 85 ], "backend": "ort", "elements": 680, "expected_dtype": "float32", "expected_shape": [ 8, 85 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 85 ], "backend": "ort", "elements": 340, "expected_dtype": "float32", "expected_shape": [ 4, 85 ], "first_mismatch": null, "max_abs": 2.1815820869163406e-39, "max_budget_ratio": 2.1815820869163406e-33, "max_rel": 9.804537851970274e+268, "max_ulp": 1556829, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } } ], "comparison": { "actual_dtype": "float32", "actual_shape": [ 4, 85 ], "elements": 340, "expected_dtype": "float32", "expected_shape": [ 4, 85 ], "first_mismatch": null, "max_abs": 3.2311742677852644e-27, "max_budget_ratio": 3.2311742677852644e-21, "max_rel": 1.0, "max_ulp": 1556829, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, "comparison_key": "f48529acaee7047a3509a49864db10741730b6648c62782dcf05b71da5bb1c28", "contract": "probability", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "float32", "shape": [ 4, 85 ] } }, "ir_version": 10, "name": "Softmax/random-021/float32/4x85/seed-20260906", "observation_id": "fe7223396ffcd2e8221e1906303d75983e02aa864b50cc2d83eb8c10ed155a57", "operation": "Softmax", "operators": [ "Softmax" ], "opset": 19, "optimization": "all", "output_hash": "e83e6e6e55b4303cbc6eab92557d664ebb17ae7c75e8eeabff28167f4ea300af", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.003166, "p95_ms": 0.0035287, "repeats": 7, "samples_ms": [ 0.003541, 0.0035, 0.003416, 0.003166, 0.003, 0.002792, 0.002833 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.011042, "p95_ms": 0.0117999, "repeats": 7, "samples_ms": [ 0.012, 0.011209, 0.011042, 0.010583, 0.011333, 0.010958, 0.010791 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-06, "rtol": 1e-05 }, "variant": "random-021", "warnings": [ { "backend": "reference", "message": "underflow encountered in divide" }, { "backend": "reference", "message": "underflow encountered in exp" } ] }, { "artifacts": { "bundle": "observations/2be360bfde8f424274467cfb0915723891ca3a439ba7f7b429e89e0fc9485e59/reproducer.zip", "bundle_sha256": "2939d81ea68070c0cb31d01886464dedba07aeac928f9406c04b55f759347f12", "case": "artifacts/6e739d54f1eae43e8852f197450cf3c27d6a0ed62fab0cc7526e0699c3c9324c/case.json", "inputs": "artifacts/6e739d54f1eae43e8852f197450cf3c27d6a0ed62fab0cc7526e0699c3c9324c/inputs.npz", "manifest": "artifacts/6e739d54f1eae43e8852f197450cf3c27d6a0ed62fab0cc7526e0699c3c9324c/manifest.json", "model": "artifacts/6e739d54f1eae43e8852f197450cf3c27d6a0ed62fab0cc7526e0699c3c9324c/model.onnx", "outputs": "observations/2be360bfde8f424274467cfb0915723891ca3a439ba7f7b429e89e0fc9485e59/outputs.npz" }, "candidate_group": "aec977c93f96d9177a5b9570d1a9a8f34a5f159f8959977d62a2091818bf0db4", "case_id": "6e739d54f1eae43e8852f197450cf3c27d6a0ed62fab0cc7526e0699c3c9324c", "category": "softmax", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "backend": "reference", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 7.194103135432783e-08, "max_budget_ratio": 0.006540093759484347, "max_rel": 7.194103135432783e-08, "max_ulp": 489357312, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 85 ], "backend": "reference", "elements": 340, "expected_dtype": "float32", "expected_shape": [ 4, 85 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 85 ], "backend": "reference", "elements": 340, "expected_dtype": "float32", "expected_shape": [ 4, 85 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 85 ], "backend": "reference", "elements": 340, "expected_dtype": "float32", "expected_shape": [ 4, 85 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 85 ], "backend": "reference", "elements": 680, "expected_dtype": "float32", "expected_shape": [ 8, 85 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 85 ], "backend": "reference", "elements": 340, "expected_dtype": "float32", "expected_shape": [ 4, 85 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "backend": "ort", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 7.194103135432783e-08, "max_budget_ratio": 0.006540093759484347, "max_rel": 7.194103135432783e-08, "max_ulp": 489357312, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 85 ], "backend": "ort", "elements": 340, "expected_dtype": "float32", "expected_shape": [ 4, 85 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 85 ], "backend": "ort", "elements": 340, "expected_dtype": "float32", "expected_shape": [ 4, 85 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 85 ], "backend": "ort", "elements": 340, "expected_dtype": "float32", "expected_shape": [ 4, 85 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 85 ], "backend": "ort", "elements": 680, "expected_dtype": "float32", "expected_shape": [ 8, 85 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 85 ], "backend": "ort", "elements": 340, "expected_dtype": "float32", "expected_shape": [ 4, 85 ], "first_mismatch": null, "max_abs": 2.1815820869163406e-39, "max_budget_ratio": 2.1815820869163406e-33, "max_rel": 9.804537851970274e+268, "max_ulp": 1556829, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } } ], "comparison": { "actual_dtype": "float32", "actual_shape": [ 4, 85 ], "elements": 340, "expected_dtype": "float32", "expected_shape": [ 4, 85 ], "first_mismatch": null, "max_abs": 3.2311742677852644e-27, "max_budget_ratio": 3.2311742677852644e-21, "max_rel": 1.0, "max_ulp": 1556829, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, "comparison_key": "72e9dda7cb1dd0997f68f1713f546af4d0a1e7e13844cb2a4acdbbe2adc8550c", "contract": "probability", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "float32", "shape": [ 4, 85 ] } }, "ir_version": 10, "name": "Softmax/random-021/float32/4x85/seed-20260906", "observation_id": "2be360bfde8f424274467cfb0915723891ca3a439ba7f7b429e89e0fc9485e59", "operation": "Softmax", "operators": [ "Softmax" ], "opset": 19, "optimization": "disabled", "output_hash": "e83e6e6e55b4303cbc6eab92557d664ebb17ae7c75e8eeabff28167f4ea300af", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.003084, "p95_ms": 0.0036211, "repeats": 7, "samples_ms": [ 0.00325, 0.003709, 0.003084, 0.003416, 0.003, 0.002875, 0.002958 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.011209, "p95_ms": 0.06644549999999994, "repeats": 7, "samples_ms": [ 0.012458, 0.011458, 0.011209, 0.011125, 0.010959, 0.011, 0.089583 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-06, "rtol": 1e-05 }, "variant": "random-021", "warnings": [ { "backend": "reference", "message": "underflow encountered in divide" }, { "backend": "reference", "message": "underflow encountered in exp" } ] }, { "artifacts": { "bundle": "observations/2678b0ae50b56a07e133e28225d982607ad56d528fe8c580401bd3affb9e6613/reproducer.zip", "bundle_sha256": "a90040f4ca36b92ecf9c61288578ba5d0c0b23dcb029394bcdd7f2e1c725ca11", "case": "artifacts/fc9feb2f7682fe38422ac25378995ec44333c36b82e5b4229a36ecb9f57b29d5/case.json", "inputs": "artifacts/fc9feb2f7682fe38422ac25378995ec44333c36b82e5b4229a36ecb9f57b29d5/inputs.npz", "manifest": "artifacts/fc9feb2f7682fe38422ac25378995ec44333c36b82e5b4229a36ecb9f57b29d5/manifest.json", "model": "artifacts/fc9feb2f7682fe38422ac25378995ec44333c36b82e5b4229a36ecb9f57b29d5/model.onnx", "outputs": "observations/2678b0ae50b56a07e133e28225d982607ad56d528fe8c580401bd3affb9e6613/outputs.npz" }, "candidate_group": "55276623a2fb5effd32a81449f4a3067ed44cfacb963b4696b077f41932b2002", "case_id": "fc9feb2f7682fe38422ac25378995ec44333c36b82e5b4229a36ecb9f57b29d5", "category": "softmax", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "backend": "reference", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 9.1552734375e-05, "max_budget_ratio": 0.016645951704545456, "max_rel": 9.1552734375e-05, "max_ulp": 412316860416, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 8, 17 ], "backend": "reference", "elements": 136, "expected_dtype": "float16", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "backend": "ort", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 3.0517578125e-05, "max_budget_ratio": 0.005548650568181819, "max_rel": 3.0517578125e-05, "max_ulp": 137438953472, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 8, 17 ], "backend": "ort", "elements": 136, "expected_dtype": "float16", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } } ], "comparison": { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 3.0517578125e-05, "max_budget_ratio": 0.0384260682446972, "max_rel": 0.0005186721991701245, "max_ulp": 1, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, "comparison_key": "9eaf25a811e03721913cdcf5db2f13881a96fd06dbd18d3309d02d33a13f5964", "contract": "probability", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "float16", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "Softmax/tiny/float16/4x17/seed-20260906", "observation_id": "2678b0ae50b56a07e133e28225d982607ad56d528fe8c580401bd3affb9e6613", "operation": "Softmax", "operators": [ "Softmax" ], "opset": 19, "optimization": "all", "output_hash": "bae275c4ef2e43a1be1543cafd1d7a9b5630a79a9f73bb59facc09bb7a7df94a", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.004833, "p95_ms": 0.005208, "repeats": 7, "samples_ms": [ 0.005208, 0.005208, 0.004833, 0.004667, 0.004458, 0.00475, 0.004958 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.008959, "p95_ms": 0.0098752, "repeats": 7, "samples_ms": [ 0.01, 0.009584, 0.009042, 0.008875, 0.008875, 0.008959, 0.008917 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 0.0005, "rtol": 0.005 }, "variant": "tiny", "warnings": [] }, { "artifacts": { "bundle": "observations/1934546c3af1b052b5a9ec004d8bb3f1b2effd3bd7954cac1f41a617f2f720c4/reproducer.zip", "bundle_sha256": "efd298c8bd76cccebe0df2d806af9295f690e7736c762c4e6e61b0806b0ba05e", "case": "artifacts/fc9feb2f7682fe38422ac25378995ec44333c36b82e5b4229a36ecb9f57b29d5/case.json", "inputs": "artifacts/fc9feb2f7682fe38422ac25378995ec44333c36b82e5b4229a36ecb9f57b29d5/inputs.npz", "manifest": "artifacts/fc9feb2f7682fe38422ac25378995ec44333c36b82e5b4229a36ecb9f57b29d5/manifest.json", "model": "artifacts/fc9feb2f7682fe38422ac25378995ec44333c36b82e5b4229a36ecb9f57b29d5/model.onnx", "outputs": "observations/1934546c3af1b052b5a9ec004d8bb3f1b2effd3bd7954cac1f41a617f2f720c4/outputs.npz" }, "candidate_group": "55276623a2fb5effd32a81449f4a3067ed44cfacb963b4696b077f41932b2002", "case_id": "fc9feb2f7682fe38422ac25378995ec44333c36b82e5b4229a36ecb9f57b29d5", "category": "softmax", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "backend": "reference", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 9.1552734375e-05, "max_budget_ratio": 0.016645951704545456, "max_rel": 9.1552734375e-05, "max_ulp": 412316860416, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 8, 17 ], "backend": "reference", "elements": 136, "expected_dtype": "float16", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "backend": "ort", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 3.0517578125e-05, "max_budget_ratio": 0.005548650568181819, "max_rel": 3.0517578125e-05, "max_ulp": 137438953472, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 8, 17 ], "backend": "ort", "elements": 136, "expected_dtype": "float16", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } } ], "comparison": { "actual_dtype": "float16", "actual_shape": [ 4, 17 ], "elements": 68, "expected_dtype": "float16", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 3.0517578125e-05, "max_budget_ratio": 0.0384260682446972, "max_rel": 0.0005186721991701245, "max_ulp": 1, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, "comparison_key": "5c486519614491a6f3eb478135f272933750e56f91854fa5c4cc8409b5b5371d", "contract": "probability", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "float16", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "Softmax/tiny/float16/4x17/seed-20260906", "observation_id": "1934546c3af1b052b5a9ec004d8bb3f1b2effd3bd7954cac1f41a617f2f720c4", "operation": "Softmax", "operators": [ "Softmax" ], "opset": 19, "optimization": "disabled", "output_hash": "bae275c4ef2e43a1be1543cafd1d7a9b5630a79a9f73bb59facc09bb7a7df94a", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.005, "p95_ms": 0.0052829, "repeats": 7, "samples_ms": [ 0.005166, 0.005333, 0.005084, 0.00475, 0.004458, 0.00425, 0.005 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.009, "p95_ms": 0.0120038, "repeats": 7, "samples_ms": [ 0.012041, 0.011917, 0.009, 0.008792, 0.008792, 0.009167, 0.008709 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 0.0005, "rtol": 0.005 }, "variant": "tiny", "warnings": [] }, { "artifacts": { "bundle": "observations/fcb35a387a1873b0885a5a09ef4252cc823adba5359f431dda18c5066e0d8296/reproducer.zip", "bundle_sha256": "1916a18691d1792c7677ff2c8b7c2f0791b0e841a134a027d089e847619f30f4", "case": "artifacts/b3d6588bda4aa39712565e75636a53a40b6b35655d3b188ee8ea2d3f84ab120f/case.json", "inputs": "artifacts/b3d6588bda4aa39712565e75636a53a40b6b35655d3b188ee8ea2d3f84ab120f/inputs.npz", "manifest": "artifacts/b3d6588bda4aa39712565e75636a53a40b6b35655d3b188ee8ea2d3f84ab120f/manifest.json", "model": "artifacts/b3d6588bda4aa39712565e75636a53a40b6b35655d3b188ee8ea2d3f84ab120f/model.onnx", "outputs": "observations/fcb35a387a1873b0885a5a09ef4252cc823adba5359f431dda18c5066e0d8296/outputs.npz" }, "candidate_group": "fee783b35a1a3c0b3053ab15ff0e83ff6647f007dc35c9ad028ea6820952443f", "case_id": "b3d6588bda4aa39712565e75636a53a40b6b35655d3b188ee8ea2d3f84ab120f", "category": "softmax", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "backend": "reference", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 5.960464477539063e-08, "max_budget_ratio": 0.005418604070490056, "max_rel": 5.960464477539063e-08, "max_ulp": 536870912, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 17 ], "backend": "reference", "elements": 136, "expected_dtype": "float32", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "backend": "ort", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 1.0058283805847168e-07, "max_budget_ratio": 0.00914389436895197, "max_rel": 1.0058283805847168e-07, "max_ulp": 570425344, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 17 ], "backend": "ort", "elements": 136, "expected_dtype": "float32", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 1.1175870895385742e-08, "max_budget_ratio": 0.007036688656662734, "max_rel": 1.8999193420908637e-07, "max_ulp": 3, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } } ], "comparison": { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 3.725290298461914e-09, "max_budget_ratio": 0.002345719418376243, "max_rel": 6.334205743490385e-08, "max_ulp": 1, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, "comparison_key": "d6560bc8d1966499f0b57b73b4398375f5e61c0b037e83bb5d45f25fc6bbcf3b", "contract": "probability", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "float32", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "Softmax/tiny/float32/4x17/seed-20260906", "observation_id": "fcb35a387a1873b0885a5a09ef4252cc823adba5359f431dda18c5066e0d8296", "operation": "Softmax", "operators": [ "Softmax" ], "opset": 19, "optimization": "all", "output_hash": "3c16e51222d2db21aa169dd03b1a944dccc270bd4ca6e486a6e644f9b469bf90", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.003042, "p95_ms": 0.0034919, "repeats": 7, "samples_ms": [ 0.003042, 0.00325, 0.003542, 0.003375, 0.003, 0.003041, 0.002791 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.0075, "p95_ms": 0.0081249, "repeats": 7, "samples_ms": [ 0.00825, 0.007833, 0.0075, 0.007375, 0.00725, 0.007375, 0.007833 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-06, "rtol": 1e-05 }, "variant": "tiny", "warnings": [] }, { "artifacts": { "bundle": "observations/8ede44a7f8a684a36377f1a6b507e5f7f0d1713b61fd506cfc2089a5c1e55d3a/reproducer.zip", "bundle_sha256": "4ee5251aa132a388c180bd45f2241a6ebd9c15db1f2856ed5318115d2021dcdf", "case": "artifacts/b3d6588bda4aa39712565e75636a53a40b6b35655d3b188ee8ea2d3f84ab120f/case.json", "inputs": "artifacts/b3d6588bda4aa39712565e75636a53a40b6b35655d3b188ee8ea2d3f84ab120f/inputs.npz", "manifest": "artifacts/b3d6588bda4aa39712565e75636a53a40b6b35655d3b188ee8ea2d3f84ab120f/manifest.json", "model": "artifacts/b3d6588bda4aa39712565e75636a53a40b6b35655d3b188ee8ea2d3f84ab120f/model.onnx", "outputs": "observations/8ede44a7f8a684a36377f1a6b507e5f7f0d1713b61fd506cfc2089a5c1e55d3a/outputs.npz" }, "candidate_group": "fee783b35a1a3c0b3053ab15ff0e83ff6647f007dc35c9ad028ea6820952443f", "case_id": "b3d6588bda4aa39712565e75636a53a40b6b35655d3b188ee8ea2d3f84ab120f", "category": "softmax", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "backend": "reference", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 5.960464477539063e-08, "max_budget_ratio": 0.005418604070490056, "max_rel": 5.960464477539063e-08, "max_ulp": 536870912, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 17 ], "backend": "reference", "elements": 136, "expected_dtype": "float32", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "backend": "ort", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 1.0058283805847168e-07, "max_budget_ratio": 0.00914389436895197, "max_rel": 1.0058283805847168e-07, "max_ulp": 570425344, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 17 ], "backend": "ort", "elements": 136, "expected_dtype": "float32", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 1.1175870895385742e-08, "max_budget_ratio": 0.007036688656662734, "max_rel": 1.8999193420908637e-07, "max_ulp": 3, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } } ], "comparison": { "actual_dtype": "float32", "actual_shape": [ 4, 17 ], "elements": 68, "expected_dtype": "float32", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 3.725290298461914e-09, "max_budget_ratio": 0.002345719418376243, "max_rel": 6.334205743490385e-08, "max_ulp": 1, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, "comparison_key": "f909d0b39981454c20c5b6c7d90e594d1d9a120cc4a0126a4aa010d56c1b925b", "contract": "probability", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "float32", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "Softmax/tiny/float32/4x17/seed-20260906", "observation_id": "8ede44a7f8a684a36377f1a6b507e5f7f0d1713b61fd506cfc2089a5c1e55d3a", "operation": "Softmax", "operators": [ "Softmax" ], "opset": 19, "optimization": "disabled", "output_hash": "3c16e51222d2db21aa169dd03b1a944dccc270bd4ca6e486a6e644f9b469bf90", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.002792, "p95_ms": 0.0031704999999999997, "repeats": 7, "samples_ms": [ 0.002834, 0.003083, 0.002667, 0.00275, 0.002792, 0.003208, 0.002666 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.008208, "p95_ms": 0.0091375, "repeats": 7, "samples_ms": [ 0.00925, 0.008875, 0.008334, 0.008208, 0.0075, 0.007583, 0.007208 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-06, "rtol": 1e-05 }, "variant": "tiny", "warnings": [] }, { "artifacts": { "bundle": "observations/d6497f5a9ae5bd5fb0a147549061516f3d623e3a553f630a36c02e311aadb7ff/reproducer.zip", "bundle_sha256": "cd8826d2e02281b1b21756ab0f802a0c20ed10890d09862f90a8c73a928f172a", "case": "artifacts/c4e0ae3d2d537b8b94c3b2e16609133fc25cac07d916abae9a8f8da46177a99c/case.json", "inputs": "artifacts/c4e0ae3d2d537b8b94c3b2e16609133fc25cac07d916abae9a8f8da46177a99c/inputs.npz", "manifest": "artifacts/c4e0ae3d2d537b8b94c3b2e16609133fc25cac07d916abae9a8f8da46177a99c/manifest.json", "model": "artifacts/c4e0ae3d2d537b8b94c3b2e16609133fc25cac07d916abae9a8f8da46177a99c/model.onnx", "outputs": "observations/d6497f5a9ae5bd5fb0a147549061516f3d623e3a553f630a36c02e311aadb7ff/outputs.npz" }, "candidate_group": "392f723b94fdb780057faf5fb5e9f999225f471fe8a57a4747597f4dab5af6ab", "case_id": "c4e0ae3d2d537b8b94c3b2e16609133fc25cac07d916abae9a8f8da46177a99c", "category": "softmax", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "backend": "reference", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 8, 17 ], "backend": "reference", "elements": 136, "expected_dtype": "float64", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "backend": "ort", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 8, 17 ], "backend": "ort", "elements": 136, "expected_dtype": "float64", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } } ], "comparison": { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 6.938893903907228e-18, "max_budget_ratio": 4.36923567600581e-05, "max_rel": 1.1798324955350365e-16, "max_ulp": 1, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, "comparison_key": "37c28351d3f0f728be730c6e824b26f59818a71f0105e70b5d5332b6dd065287", "contract": "probability", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "float64", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "Softmax/tiny/float64/4x17/seed-20260906", "observation_id": "d6497f5a9ae5bd5fb0a147549061516f3d623e3a553f630a36c02e311aadb7ff", "operation": "Softmax", "operators": [ "Softmax" ], "opset": 19, "optimization": "all", "output_hash": "da9890b31d073adebb2b4aa72508fbd92e7ff340709bfa7331c7e665b5ecaea4", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.003333, "p95_ms": 0.0035794, "repeats": 7, "samples_ms": [ 0.003375, 0.003667, 0.003333, 0.003334, 0.00325, 0.003125, 0.002917 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.008333, "p95_ms": 0.009071300000000001, "repeats": 7, "samples_ms": [ 0.009209, 0.00875, 0.008375, 0.008208, 0.007958, 0.008333, 0.007958 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-13, "rtol": 1e-12 }, "variant": "tiny", "warnings": [] }, { "artifacts": { "bundle": "observations/384fcdfdc2ae5a60399e48f93a975aea4fee26e8780d7ab7d96ce32df019fa2d/reproducer.zip", "bundle_sha256": "3cccbe0dc067d4d1255a6a77e5227c0880d300a89eb1929142e67a1d51316a57", "case": "artifacts/c4e0ae3d2d537b8b94c3b2e16609133fc25cac07d916abae9a8f8da46177a99c/case.json", "inputs": "artifacts/c4e0ae3d2d537b8b94c3b2e16609133fc25cac07d916abae9a8f8da46177a99c/inputs.npz", "manifest": "artifacts/c4e0ae3d2d537b8b94c3b2e16609133fc25cac07d916abae9a8f8da46177a99c/manifest.json", "model": "artifacts/c4e0ae3d2d537b8b94c3b2e16609133fc25cac07d916abae9a8f8da46177a99c/model.onnx", "outputs": "observations/384fcdfdc2ae5a60399e48f93a975aea4fee26e8780d7ab7d96ce32df019fa2d/outputs.npz" }, "candidate_group": "392f723b94fdb780057faf5fb5e9f999225f471fe8a57a4747597f4dab5af6ab", "case_id": "c4e0ae3d2d537b8b94c3b2e16609133fc25cac07d916abae9a8f8da46177a99c", "category": "softmax", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "backend": "reference", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 8, 17 ], "backend": "reference", "elements": 136, "expected_dtype": "float64", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "reference", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "backend": "ort", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 8, 17 ], "backend": "ort", "elements": 136, "expected_dtype": "float64", "expected_shape": [ 8, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "backend": "ort", "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } } ], "comparison": { "actual_dtype": "float64", "actual_shape": [ 4, 17 ], "elements": 68, "expected_dtype": "float64", "expected_shape": [ 4, 17 ], "first_mismatch": null, "max_abs": 6.938893903907228e-18, "max_budget_ratio": 4.36923567600581e-05, "max_rel": 1.1798324955350365e-16, "max_ulp": 1, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, "comparison_key": "5a00a43137ba0d5a18b7dcb32f8ed670c4ea6bfcc0ac096c5a5b2ada179aaadd", "contract": "probability", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "float64", "shape": [ 4, 17 ] } }, "ir_version": 10, "name": "Softmax/tiny/float64/4x17/seed-20260906", "observation_id": "384fcdfdc2ae5a60399e48f93a975aea4fee26e8780d7ab7d96ce32df019fa2d", "operation": "Softmax", "operators": [ "Softmax" ], "opset": 19, "optimization": "disabled", "output_hash": "da9890b31d073adebb2b4aa72508fbd92e7ff340709bfa7331c7e665b5ecaea4", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.003417, "p95_ms": 0.0041252, "repeats": 7, "samples_ms": [ 0.0035, 0.00425, 0.003834, 0.003416, 0.003375, 0.003417, 0.002959 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.0075, "p95_ms": 0.008225, "repeats": 7, "samples_ms": [ 0.008375, 0.007875, 0.007542, 0.007333, 0.007375, 0.0075, 0.00725 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-13, "rtol": 1e-12 }, "variant": "tiny", "warnings": [] }, { "artifacts": { "bundle": "observations/4d53206d2b86a662b6c33aff96f103ac21f16ff50792b95278ebea94e51fb183/reproducer.zip", "bundle_sha256": "67d0581720dd79226520c042f21ee7d5f9ae2a21843270e7bf74b4a549c1d25f", "case": "artifacts/3ecd92906b34e6950ee3d553791b66f770c1f8efc49df916f17e2ed5f278e502/case.json", "inputs": "artifacts/3ecd92906b34e6950ee3d553791b66f770c1f8efc49df916f17e2ed5f278e502/inputs.npz", "manifest": "artifacts/3ecd92906b34e6950ee3d553791b66f770c1f8efc49df916f17e2ed5f278e502/manifest.json", "model": "artifacts/3ecd92906b34e6950ee3d553791b66f770c1f8efc49df916f17e2ed5f278e502/model.onnx", "outputs": "observations/4d53206d2b86a662b6c33aff96f103ac21f16ff50792b95278ebea94e51fb183/outputs.npz" }, "candidate_group": "51fb5a10f530edbb3e270a08cdf703b037a62d9f5aa727e28bb13640ecf0dd00", "case_id": "3ecd92906b34e6950ee3d553791b66f770c1f8efc49df916f17e2ed5f278e502", "category": "softmax", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "backend": "reference", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 0.00028192996978759766, "max_budget_ratio": 0.05125999450683594, "max_rel": 0.00028192996978759766, "max_ulp": 1269699706880, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 257 ], "backend": "reference", "elements": 1028, "expected_dtype": "float16", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 257 ], "backend": "reference", "elements": 1028, "expected_dtype": "float16", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 257 ], "backend": "reference", "elements": 1028, "expected_dtype": "float16", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 8, 257 ], "backend": "reference", "elements": 2056, "expected_dtype": "float16", "expected_shape": [ 8, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 257 ], "backend": "reference", "elements": 1028, "expected_dtype": "float16", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "backend": "ort", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 1.7762184143066406e-05, "max_budget_ratio": 0.003229488026012074, "max_rel": 1.7762184143066406e-05, "max_ulp": 155692564480, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 257 ], "backend": "ort", "elements": 1028, "expected_dtype": "float16", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 257 ], "backend": "ort", "elements": 1028, "expected_dtype": "float16", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 257 ], "backend": "ort", "elements": 1028, "expected_dtype": "float16", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 8, 257 ], "backend": "ort", "elements": 2056, "expected_dtype": "float16", "expected_shape": [ 8, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 257 ], "backend": "ort", "elements": 1028, "expected_dtype": "float16", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } } ], "comparison": { "actual_dtype": "float16", "actual_shape": [ 4, 257 ], "elements": 1028, "expected_dtype": "float16", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 1.52587890625e-05, "max_budget_ratio": 0.02626533238778137, "max_rel": 0.0028591851322373124, "max_ulp": 5, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, "comparison_key": "7b6394b8876a7e5681082a0d906f5cd1336cd6d70b00887a472a75a91af20226", "contract": "probability", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "float16", "shape": [ 4, 257 ] } }, "ir_version": 10, "name": "Softmax/wide/float16/4x257/seed-20260906", "observation_id": "4d53206d2b86a662b6c33aff96f103ac21f16ff50792b95278ebea94e51fb183", "operation": "Softmax", "operators": [ "Softmax" ], "opset": 19, "optimization": "all", "output_hash": "6800d0e572172dba4ba1777c4e5eebfd7883889edfc6c37883445ccdefee8c95", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.006125, "p95_ms": 0.0067044, "repeats": 7, "samples_ms": [ 0.0065, 0.006458, 0.006125, 0.006792, 0.0055, 0.005792, 0.005208 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.020917, "p95_ms": 0.021224800000000002, "repeats": 7, "samples_ms": [ 0.019958, 0.02125, 0.021, 0.020917, 0.020875, 0.021166, 0.020625 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 0.0005, "rtol": 0.005 }, "variant": "wide", "warnings": [] }, { "artifacts": { "bundle": "observations/9f1ce224c55066511701baf87ce544bbb234ff726c37e393d8ce7f03ff48b78e/reproducer.zip", "bundle_sha256": "5e71ac69021af83e568142d112fce739a2e19df795b9d1124c61d95030ce1733", "case": "artifacts/3ecd92906b34e6950ee3d553791b66f770c1f8efc49df916f17e2ed5f278e502/case.json", "inputs": "artifacts/3ecd92906b34e6950ee3d553791b66f770c1f8efc49df916f17e2ed5f278e502/inputs.npz", "manifest": "artifacts/3ecd92906b34e6950ee3d553791b66f770c1f8efc49df916f17e2ed5f278e502/manifest.json", "model": "artifacts/3ecd92906b34e6950ee3d553791b66f770c1f8efc49df916f17e2ed5f278e502/model.onnx", "outputs": "observations/9f1ce224c55066511701baf87ce544bbb234ff726c37e393d8ce7f03ff48b78e/outputs.npz" }, "candidate_group": "51fb5a10f530edbb3e270a08cdf703b037a62d9f5aa727e28bb13640ecf0dd00", "case_id": "3ecd92906b34e6950ee3d553791b66f770c1f8efc49df916f17e2ed5f278e502", "category": "softmax", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "backend": "reference", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 0.00028192996978759766, "max_budget_ratio": 0.05125999450683594, "max_rel": 0.00028192996978759766, "max_ulp": 1269699706880, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 257 ], "backend": "reference", "elements": 1028, "expected_dtype": "float16", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 257 ], "backend": "reference", "elements": 1028, "expected_dtype": "float16", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 257 ], "backend": "reference", "elements": 1028, "expected_dtype": "float16", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 8, 257 ], "backend": "reference", "elements": 2056, "expected_dtype": "float16", "expected_shape": [ 8, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 257 ], "backend": "reference", "elements": 1028, "expected_dtype": "float16", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "backend": "ort", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 1.7762184143066406e-05, "max_budget_ratio": 0.003229488026012074, "max_rel": 1.7762184143066406e-05, "max_ulp": 155692564480, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 257 ], "backend": "ort", "elements": 1028, "expected_dtype": "float16", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 257 ], "backend": "ort", "elements": 1028, "expected_dtype": "float16", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 257 ], "backend": "ort", "elements": 1028, "expected_dtype": "float16", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 8, 257 ], "backend": "ort", "elements": 2056, "expected_dtype": "float16", "expected_shape": [ 8, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, { "actual_dtype": "float16", "actual_shape": [ 4, 257 ], "backend": "ort", "elements": 1028, "expected_dtype": "float16", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } } ], "comparison": { "actual_dtype": "float16", "actual_shape": [ 4, 257 ], "elements": 1028, "expected_dtype": "float16", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 1.52587890625e-05, "max_budget_ratio": 0.02626533238778137, "max_rel": 0.0028591851322373124, "max_ulp": 5, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 0.0005, "rtol": 0.005 } }, "comparison_key": "5ba4839cbd9e7d50eefe3dbdbf356bf5ddbd6cbeb77611f073481e6d062cd05a", "contract": "probability", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "float16", "shape": [ 4, 257 ] } }, "ir_version": 10, "name": "Softmax/wide/float16/4x257/seed-20260906", "observation_id": "9f1ce224c55066511701baf87ce544bbb234ff726c37e393d8ce7f03ff48b78e", "operation": "Softmax", "operators": [ "Softmax" ], "opset": 19, "optimization": "disabled", "output_hash": "6800d0e572172dba4ba1777c4e5eebfd7883889edfc6c37883445ccdefee8c95", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.006334, "p95_ms": 0.009262299999999998, "repeats": 7, "samples_ms": [ 0.006666, 0.006375, 0.006292, 0.010375, 0.006334, 0.005542, 0.005792 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.020792, "p95_ms": 0.021209, "repeats": 7, "samples_ms": [ 0.02025, 0.019459, 0.020542, 0.021209, 0.020875, 0.021209, 0.020792 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 0.0005, "rtol": 0.005 }, "variant": "wide", "warnings": [] }, { "artifacts": { "bundle": "observations/187eb6d04324fcb61117eaf9a4875103c39b9d902f368a4679bb476a11af84e3/reproducer.zip", "bundle_sha256": "24179af640682b2043ba825f5833b17c3ea9885fd0362e96af774f1b61d8006d", "case": "artifacts/9bd45b09728c3ccb967fd98f823eeccfe22a42b01d415cc4af7398f238312414/case.json", "inputs": "artifacts/9bd45b09728c3ccb967fd98f823eeccfe22a42b01d415cc4af7398f238312414/inputs.npz", "manifest": "artifacts/9bd45b09728c3ccb967fd98f823eeccfe22a42b01d415cc4af7398f238312414/manifest.json", "model": "artifacts/9bd45b09728c3ccb967fd98f823eeccfe22a42b01d415cc4af7398f238312414/model.onnx", "outputs": "observations/187eb6d04324fcb61117eaf9a4875103c39b9d902f368a4679bb476a11af84e3/outputs.npz" }, "candidate_group": "bc7fa4734253c810c670e8cf43079e5a60153ae92cda0f7628c5da56d1dfe4d2", "case_id": "9bd45b09728c3ccb967fd98f823eeccfe22a42b01d415cc4af7398f238312414", "category": "softmax", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "backend": "reference", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 7.29633029550314e-08, "max_budget_ratio": 0.0066330275413664895, "max_rel": 7.29633029550314e-08, "max_ulp": 657195008, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 257 ], "backend": "reference", "elements": 1028, "expected_dtype": "float32", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 257 ], "backend": "reference", "elements": 1028, "expected_dtype": "float32", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 257 ], "backend": "reference", "elements": 1028, "expected_dtype": "float32", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 257 ], "backend": "reference", "elements": 2056, "expected_dtype": "float32", "expected_shape": [ 8, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 257 ], "backend": "reference", "elements": 1028, "expected_dtype": "float32", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "backend": "ort", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 1.0018266038969159e-07, "max_budget_ratio": 0.009107514580881052, "max_rel": 1.0018266038969159e-07, "max_ulp": 902365184, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 257 ], "backend": "ort", "elements": 1028, "expected_dtype": "float32", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 257 ], "backend": "ort", "elements": 1028, "expected_dtype": "float32", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 257 ], "backend": "ort", "elements": 1028, "expected_dtype": "float32", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 257 ], "backend": "ort", "elements": 2056, "expected_dtype": "float32", "expected_shape": [ 8, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 257 ], "backend": "ort", "elements": 1028, "expected_dtype": "float32", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 3.725290298461914e-09, "max_budget_ratio": 0.003178744362028009, "max_rel": 2.3671305927010948e-07, "max_ulp": 3, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } } ], "comparison": { "actual_dtype": "float32", "actual_shape": [ 4, 257 ], "elements": 1028, "expected_dtype": "float32", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 7.450580596923828e-09, "max_budget_ratio": 0.005378657978900525, "max_rel": 2.639498833077566e-07, "max_ulp": 4, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, "comparison_key": "8e7572b8a4d2ae5e9b92e53c156d5081c82fea6cd64187f05862f103b4572b22", "contract": "probability", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "float32", "shape": [ 4, 257 ] } }, "ir_version": 10, "name": "Softmax/wide/float32/4x257/seed-20260906", "observation_id": "187eb6d04324fcb61117eaf9a4875103c39b9d902f368a4679bb476a11af84e3", "operation": "Softmax", "operators": [ "Softmax" ], "opset": 19, "optimization": "all", "output_hash": "3bcb88b4400c00d2e9f9bee76806995d9a602d521afe180b08720fc55e863ba9", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.003292, "p95_ms": 0.0035749, "repeats": 7, "samples_ms": [ 0.003417, 0.003625, 0.003458, 0.003292, 0.00325, 0.003209, 0.00325 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.010083, "p95_ms": 0.0108965, "repeats": 7, "samples_ms": [ 0.011084, 0.010459, 0.010083, 0.010084, 0.009958, 0.010042, 0.009791 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-06, "rtol": 1e-05 }, "variant": "wide", "warnings": [] }, { "artifacts": { "bundle": "observations/4dc39c6103641903705c7845c59af4f2c7c844254a3b2db902cb1ab587351893/reproducer.zip", "bundle_sha256": "a415befcaad7c09ce2ebcd0f13fde75e86b87a7cf70d969a744f129f8e259e16", "case": "artifacts/9bd45b09728c3ccb967fd98f823eeccfe22a42b01d415cc4af7398f238312414/case.json", "inputs": "artifacts/9bd45b09728c3ccb967fd98f823eeccfe22a42b01d415cc4af7398f238312414/inputs.npz", "manifest": "artifacts/9bd45b09728c3ccb967fd98f823eeccfe22a42b01d415cc4af7398f238312414/manifest.json", "model": "artifacts/9bd45b09728c3ccb967fd98f823eeccfe22a42b01d415cc4af7398f238312414/model.onnx", "outputs": "observations/4dc39c6103641903705c7845c59af4f2c7c844254a3b2db902cb1ab587351893/outputs.npz" }, "candidate_group": "bc7fa4734253c810c670e8cf43079e5a60153ae92cda0f7628c5da56d1dfe4d2", "case_id": "9bd45b09728c3ccb967fd98f823eeccfe22a42b01d415cc4af7398f238312414", "category": "softmax", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "backend": "reference", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 7.29633029550314e-08, "max_budget_ratio": 0.0066330275413664895, "max_rel": 7.29633029550314e-08, "max_ulp": 657195008, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 257 ], "backend": "reference", "elements": 1028, "expected_dtype": "float32", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 257 ], "backend": "reference", "elements": 1028, "expected_dtype": "float32", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 257 ], "backend": "reference", "elements": 1028, "expected_dtype": "float32", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 257 ], "backend": "reference", "elements": 2056, "expected_dtype": "float32", "expected_shape": [ 8, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 257 ], "backend": "reference", "elements": 1028, "expected_dtype": "float32", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "backend": "ort", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 1.0018266038969159e-07, "max_budget_ratio": 0.009107514580881052, "max_rel": 1.0018266038969159e-07, "max_ulp": 902365184, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 257 ], "backend": "ort", "elements": 1028, "expected_dtype": "float32", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 257 ], "backend": "ort", "elements": 1028, "expected_dtype": "float32", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 257 ], "backend": "ort", "elements": 1028, "expected_dtype": "float32", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 8, 257 ], "backend": "ort", "elements": 2056, "expected_dtype": "float32", "expected_shape": [ 8, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, { "actual_dtype": "float32", "actual_shape": [ 4, 257 ], "backend": "ort", "elements": 1028, "expected_dtype": "float32", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 3.725290298461914e-09, "max_budget_ratio": 0.003178744362028009, "max_rel": 2.3671305927010948e-07, "max_ulp": 3, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } } ], "comparison": { "actual_dtype": "float32", "actual_shape": [ 4, 257 ], "elements": 1028, "expected_dtype": "float32", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 7.450580596923828e-09, "max_budget_ratio": 0.005378657978900525, "max_rel": 2.639498833077566e-07, "max_ulp": 4, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-06, "rtol": 1e-05 } }, "comparison_key": "b3f8efb41092e5504ce091e7e00064227f4e8a9b6b95fabe09b718cf10b328f9", "contract": "probability", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "float32", "shape": [ 4, 257 ] } }, "ir_version": 10, "name": "Softmax/wide/float32/4x257/seed-20260906", "observation_id": "4dc39c6103641903705c7845c59af4f2c7c844254a3b2db902cb1ab587351893", "operation": "Softmax", "operators": [ "Softmax" ], "opset": 19, "optimization": "disabled", "output_hash": "3bcb88b4400c00d2e9f9bee76806995d9a602d521afe180b08720fc55e863ba9", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.003625, "p95_ms": 0.0040292999999999995, "repeats": 7, "samples_ms": [ 0.003667, 0.004167, 0.003625, 0.003708, 0.003334, 0.003333, 0.002959 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.009083, "p95_ms": 0.0100751, "repeats": 7, "samples_ms": [ 0.01025, 0.009667, 0.009084, 0.009083, 0.008916, 0.008875, 0.008875 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-06, "rtol": 1e-05 }, "variant": "wide", "warnings": [] }, { "artifacts": { "bundle": "observations/305c6f62ee849a426c8ad383098f79495b377a126982efdaa9f758e3b1f2d6e6/reproducer.zip", "bundle_sha256": "70e6d76a1d83e7190b15da1080f04b5e56a6ca43c817853c669f53ddaef96afc", "case": "artifacts/abddc7afb5bb3e0d2c75a2c51358f8fda08377e45a48cd5ac9633d9f775988e6/case.json", "inputs": "artifacts/abddc7afb5bb3e0d2c75a2c51358f8fda08377e45a48cd5ac9633d9f775988e6/inputs.npz", "manifest": "artifacts/abddc7afb5bb3e0d2c75a2c51358f8fda08377e45a48cd5ac9633d9f775988e6/manifest.json", "model": "artifacts/abddc7afb5bb3e0d2c75a2c51358f8fda08377e45a48cd5ac9633d9f775988e6/model.onnx", "outputs": "observations/305c6f62ee849a426c8ad383098f79495b377a126982efdaa9f758e3b1f2d6e6/outputs.npz" }, "candidate_group": "92ae48016fe71f55db3c08c5373fbf802c1ffe91ad973f1a5d25d9143526206b", "case_id": "abddc7afb5bb3e0d2c75a2c51358f8fda08377e45a48cd5ac9633d9f775988e6", "category": "softmax", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "backend": "reference", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 2.220446049250313e-16, "max_budget_ratio": 0.00020185873175002848, "max_rel": 2.220446049250313e-16, "max_ulp": 2, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 257 ], "backend": "reference", "elements": 1028, "expected_dtype": "float64", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 257 ], "backend": "reference", "elements": 1028, "expected_dtype": "float64", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 257 ], "backend": "reference", "elements": 1028, "expected_dtype": "float64", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 8, 257 ], "backend": "reference", "elements": 2056, "expected_dtype": "float64", "expected_shape": [ 8, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 257 ], "backend": "reference", "elements": 1028, "expected_dtype": "float64", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "backend": "ort", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 4.440892098500626e-16, "max_budget_ratio": 0.00040371746350005696, "max_rel": 4.440892098500626e-16, "max_ulp": 2, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 257 ], "backend": "ort", "elements": 1028, "expected_dtype": "float64", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 1.0408340855860843e-17, "max_budget_ratio": 8.149316457981183e-05, "max_rel": 5.765972622988668e-16, "max_ulp": 4, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 257 ], "backend": "ort", "elements": 1028, "expected_dtype": "float64", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 257 ], "backend": "ort", "elements": 1028, "expected_dtype": "float64", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 8, 257 ], "backend": "ort", "elements": 2056, "expected_dtype": "float64", "expected_shape": [ 8, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 257 ], "backend": "ort", "elements": 1028, "expected_dtype": "float64", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 1.734723475976807e-17, "max_budget_ratio": 0.00013582194096635307, "max_rel": 7.687963497318225e-16, "max_ulp": 6, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } } ], "comparison": { "actual_dtype": "float64", "actual_shape": [ 4, 257 ], "elements": 1028, "expected_dtype": "float64", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 1.734723475976807e-17, "max_budget_ratio": 0.0001358219409663531, "max_rel": 8.633989672721415e-16, "max_ulp": 7, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, "comparison_key": "4e565b834254fec0868e642b241028a9cc80a04c1167f3a87be8a281f9e19cf5", "contract": "probability", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "float64", "shape": [ 4, 257 ] } }, "ir_version": 10, "name": "Softmax/wide/float64/4x257/seed-20260906", "observation_id": "305c6f62ee849a426c8ad383098f79495b377a126982efdaa9f758e3b1f2d6e6", "operation": "Softmax", "operators": [ "Softmax" ], "opset": 19, "optimization": "all", "output_hash": "f1046dbf4b1c263e9ea7ec5608d6f18f48e8bb979c5b7e0e2607994e070a8fa0", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.005, "p95_ms": 0.006187699999999999, "repeats": 7, "samples_ms": [ 0.00525, 0.0065, 0.005459, 0.005, 0.00475, 0.004625, 0.004708 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.010792, "p95_ms": 0.0114918, "repeats": 7, "samples_ms": [ 0.011667, 0.010792, 0.010375, 0.010208, 0.010084, 0.010833, 0.011083 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-13, "rtol": 1e-12 }, "variant": "wide", "warnings": [] }, { "artifacts": { "bundle": "observations/0f0a041fdec6097c7c2e4502c960de826805868392e9bdd5c66bda5c4f8da79d/reproducer.zip", "bundle_sha256": "c5a53426ffcce39b727f184bb94d41ab2ced44fe385bfeafc20fecf4ed42509a", "case": "artifacts/abddc7afb5bb3e0d2c75a2c51358f8fda08377e45a48cd5ac9633d9f775988e6/case.json", "inputs": "artifacts/abddc7afb5bb3e0d2c75a2c51358f8fda08377e45a48cd5ac9633d9f775988e6/inputs.npz", "manifest": "artifacts/abddc7afb5bb3e0d2c75a2c51358f8fda08377e45a48cd5ac9633d9f775988e6/manifest.json", "model": "artifacts/abddc7afb5bb3e0d2c75a2c51358f8fda08377e45a48cd5ac9633d9f775988e6/model.onnx", "outputs": "observations/0f0a041fdec6097c7c2e4502c960de826805868392e9bdd5c66bda5c4f8da79d/outputs.npz" }, "candidate_group": "92ae48016fe71f55db3c08c5373fbf802c1ffe91ad973f1a5d25d9143526206b", "case_id": "abddc7afb5bb3e0d2c75a2c51358f8fda08377e45a48cd5ac9633d9f775988e6", "category": "softmax", "checks": [ { "backend": "reference", "name": "finite_output", "passed": true }, { "backend": "reference", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "reference", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 2.220446049250313e-16, "max_budget_ratio": 0.00020185873175002848, "max_rel": 2.220446049250313e-16, "max_ulp": 2, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 257 ], "backend": "reference", "elements": 1028, "expected_dtype": "float64", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 257 ], "backend": "reference", "elements": 1028, "expected_dtype": "float64", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 257 ], "backend": "reference", "elements": 1028, "expected_dtype": "float64", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 8, 257 ], "backend": "reference", "elements": 2056, "expected_dtype": "float64", "expected_shape": [ 8, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 257 ], "backend": "reference", "elements": 1028, "expected_dtype": "float64", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "backend": "ort", "name": "finite_output", "passed": true }, { "backend": "ort", "name": "probability_range", "passed": true }, { "actual_dtype": "float64", "actual_shape": [ 4 ], "backend": "ort", "elements": 4, "expected_dtype": "float64", "expected_shape": [ 4 ], "first_mismatch": null, "max_abs": 4.440892098500626e-16, "max_budget_ratio": 0.00040371746350005696, "max_rel": 4.440892098500626e-16, "max_ulp": 2, "mismatched": 0, "name": "probability_mass", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 257 ], "backend": "ort", "elements": 1028, "expected_dtype": "float64", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 1.0408340855860843e-17, "max_budget_ratio": 8.149316457981183e-05, "max_rel": 5.765972622988668e-16, "max_ulp": 4, "mismatched": 0, "name": "batch_partition", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 257 ], "backend": "ort", "elements": 1028, "expected_dtype": "float64", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_permutation", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 257 ], "backend": "ort", "elements": 1028, "expected_dtype": "float64", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_reshape", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 8, 257 ], "backend": "ort", "elements": 2056, "expected_dtype": "float64", "expected_shape": [ 8, 257 ], "first_mismatch": null, "max_abs": 0.0, "max_budget_ratio": 0.0, "max_rel": 0.0, "max_ulp": 0, "mismatched": 0, "name": "batch_duplicate", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, { "actual_dtype": "float64", "actual_shape": [ 4, 257 ], "backend": "ort", "elements": 1028, "expected_dtype": "float64", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 1.734723475976807e-17, "max_budget_ratio": 0.00013582194096635307, "max_rel": 7.687963497318225e-16, "max_ulp": 6, "mismatched": 0, "name": "equivalent_graph", "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } } ], "comparison": { "actual_dtype": "float64", "actual_shape": [ 4, 257 ], "elements": 1028, "expected_dtype": "float64", "expected_shape": [ 4, 257 ], "first_mismatch": null, "max_abs": 1.734723475976807e-17, "max_budget_ratio": 0.0001358219409663531, "max_rel": 8.633989672721415e-16, "max_ulp": 7, "mismatched": 0, "nonfinite": 0, "numerical_divergence": false, "passed": true, "reason": null, "tolerance": { "atol": 1e-13, "rtol": 1e-12 } }, "comparison_key": "a06e1eada3a726327440e56e5999a346be3261c43b9028d3a3ceaec28ac62785", "contract": "probability", "environment": { "dependencies": { "flatbuffers": "25.12.19", "ml_dtypes": "0.6.0", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "packaging": "26.3", "protobuf": "7.36.1", "sympy": "1.14.0" }, "deterministic_compute": true, "gero_version": "0.1.0", "machine": "arm64", "numpy": "2.2.6", "onnx": "1.19.0", "onnxruntime": "1.22.1", "ort_build": "ORT Build Info: git-branch=HEAD, git-commit-id=89746dc19a, build type=Release, cmake cxx flags: -DNDEBUG -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -ffunction-sections -fdata-sections -fvisibility=hidden -fvisibility-inlines-hidden", "platform": "macOS-15.5-arm64-arm-64bit", "processor": "arm (CPU model unavailable)", "provider": "CPUExecutionProvider", "python": "3.12.13", "reference_optimized": false, "source_digest": "79a49f45e5b7f5c2e1a21debe1ef3d85383fcb8e6152089619073e800cac7c1f", "threads": 1 }, "errors": [], "inputs": { "x": { "dtype": "float64", "shape": [ 4, 257 ] } }, "ir_version": 10, "name": "Softmax/wide/float64/4x257/seed-20260906", "observation_id": "0f0a041fdec6097c7c2e4502c960de826805868392e9bdd5c66bda5c4f8da79d", "operation": "Softmax", "operators": [ "Softmax" ], "opset": 19, "optimization": "disabled", "output_hash": "f1046dbf4b1c263e9ea7ec5608d6f18f48e8bb979c5b7e0e2607994e070a8fa0", "outputs": {}, "parameters": {}, "regression": "not_assessed", "seed": 20260906, "status": "pass", "timings": { "ort": { "median_ms": 0.005041, "p95_ms": 0.006078799999999999, "repeats": 7, "samples_ms": [ 0.005041, 0.006416, 0.005292, 0.005208, 0.004917, 0.004875, 0.004584 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 }, "reference": { "median_ms": 0.011125, "p95_ms": 0.0115455, "repeats": 7, "samples_ms": [ 0.011583, 0.011125, 0.0105, 0.010167, 0.011125, 0.011458, 0.011084 ], "scope": "synchronous run including Python overhead; session construction excluded", "warmup": 2 } }, "tolerance": { "atol": 1e-13, "rtol": 1e-12 }, "variant": "wide", "warnings": [] } ], "run_id": "ecf59104f767379863b0c7913f410831f5fbde7515a943bb22588fcb9332eee3", "schema_version": 1, "summary": { "divergence": 14, "pass": 232 } }