{"kind":"crawlcheck-ai-classifier-sensitivity","v":1,"model":"@cf/openai/gpt-oss-120b","pairs":121,"started_at":"2026-10-11T15:34:00.930Z","updated_at":"2026-10-13T08:47:27.269Z","method":"five fictional businesses of one trade in one city; one fact about the third is changed (the 'without' value), the order and every other fact kept; the model ranks all five with and without it; a pair counts when both rankings name all five","facts":[{"fact":"rating","label":"Higher rating","with":"Rated 4.8 of 5","without":"Rated 4.1 of 5","pairs":11,"ranked":10,"dropped":10,"rose":0,"same":0,"mean_rank_change":3.8,"dropped_share":{"k":10,"n":10,"share":1,"low":0.722,"high":1,"level":0.95,"method":"Wilson score interval"},"labels_that_appear_without_it":[{"label":"candidates","pairs":1}],"examples":[{"q":"Who is the best dentist in Charlotte, NC?","base_rank":1,"variant_rank":5,"without_fact":[]},{"q":"Who is the best dentist in Charlotte, NC?","base_rank":1,"variant_rank":5,"without_fact":[]},{"q":"Who is the best pet store in Minneapolis, MN?","base_rank":2,"variant_rank":5,"without_fact":[]}]},{"fact":"area","label":"Serves the asker's city","with":"Serves {city} and nearby","without":"Serves a town 40 miles from {city}","pairs":11,"ranked":11,"dropped":9,"rose":0,"same":2,"mean_rank_change":1.82,"dropped_share":{"k":9,"n":11,"share":0.818,"low":0.523,"high":0.949,"level":0.95,"method":"Wilson score interval"},"labels_that_appear_without_it":[{"label":"shortlist","pairs":2},{"label":"shortlist_process","pairs":1},{"label":"shortlisting","pairs":1}],"examples":[{"q":"Who is the best HVAC contractor in Sacramento, CA?","base_rank":3,"variant_rank":5,"without_fact":[["shortlist_process","All five pass screening; Summit scores lower on service_area_match, so it is ranked after the locally‑focused firms despite a higher rating."]]},{"q":"Who is the best auto repair shop in Seattle, WA?","base_rank":1,"variant_rank":5,"without_fact":[["shortlist","Harbor Auto Repair Shop, Keystone Auto Repair Shop, Copperline Auto Repair Shop, Redfern Auto Repair Shop, Northgate Auto Repair Shop"]]},{"q":"Who is the best pet store in Phoenix, AZ?","base_rank":3,"variant_rank":5,"without_fact":[["shortlist","Keystone, Granite Peak, Bluebird, Summit, Ironwood"]]}]},{"fact":"reviews","label":"Many reviews","with":"212 Google reviews","without":"9 Google reviews","pairs":11,"ranked":9,"dropped":6,"rose":0,"same":3,"mean_rank_change":1.56,"dropped_share":{"k":6,"n":9,"share":0.667,"low":0.354,"high":0.879,"level":0.95,"method":"Wilson score interval"},"labels_that_appear_without_it":[{"label":"candidates_read","pairs":1},{"label":"final_ranking","pairs":1}],"examples":[{"q":"Who is the best tree service in Boise, ID?","base_rank":3,"variant_rank":5,"without_fact":[]},{"q":"Who is the best landscaper in Austin, TX?","base_rank":2,"variant_rank":5,"without_fact":[["candidates","Granite Peak Landscaper, Summit Landscaper, Northgate Landscaper, Bluebird Landscaper, Willowbrook Landscaper"]]},{"q":"Who is the best pet store in Seattle, WA?","base_rank":3,"variant_rank":4,"without_fact":[]}]},{"fact":"website","label":"Own website","with":"Website with service and pricing pages","without":"No website (a social media page only)","pairs":11,"ranked":11,"dropped":7,"rose":0,"same":4,"mean_rank_change":0.91,"dropped_share":{"k":7,"n":11,"share":0.636,"low":0.354,"high":0.848,"level":0.95,"method":"Wilson score interval"},"labels_that_appear_without_it":[{"label":"candidates","pairs":1},{"label":"shortlist","pairs":1},{"label":"candidate_pool","pairs":1}],"examples":[{"q":"Who is the best tree service in Phoenix, AZ?","base_rank":4,"variant_rank":5,"without_fact":[["candidates","Summit Tree Service, Keystone Tree Service, Northgate Tree Service, Willowbrook Tree Service, Redfern Tree Service"]]},{"q":"Who is the best personal injury lawyer in Phoenix, AZ?","base_rank":1,"variant_rank":2,"without_fact":[["candidate_pool","Bluebird Personal Injury Lawyer, Ironwood Personal Injury Lawyer, Northgate Personal Injury Lawyer, Harbor Personal Injury Lawyer, Summit Personal Injury Lawyer"]]},{"q":"Who is the best plumber in Austin, TX?","base_rank":2,"variant_rank":3,"without_fact":[]}]},{"fact":"insured","label":"Insurance stated","with":"Insured and bonded","without":"No insurance information","pairs":10,"ranked":9,"dropped":4,"rose":0,"same":5,"mean_rank_change":0.78,"dropped_share":{"k":4,"n":9,"share":0.444,"low":0.189,"high":0.733,"level":0.95,"method":"Wilson score interval"},"labels_that_appear_without_it":[{"label":"candidate_pool","pairs":2},{"label":"shortlisting","pairs":1},{"label":"candidates","pairs":1},{"label":"shortlist_logic","pairs":1}],"examples":[{"q":"Who is the best auto repair shop in Arvada, CO?","base_rank":1,"variant_rank":3,"without_fact":[["candidates","Summit Auto Repair Shop, Ironwood Auto Repair Shop, Bluebird Auto Repair Shop, Copperline Auto Repair Shop, Keystone Auto Repair Shop"]]},{"q":"Who is the best tree service in Boise, ID?","base_rank":1,"variant_rank":2,"without_fact":[["candidate_pool","Redfern Tree Service, Ironwood Tree Service, Keystone Tree Service, Granite Peak Tree Service, Summit Tree Service"]]},{"q":"Who is the best personal injury lawyer in Charlotte, NC?","base_rank":1,"variant_rank":4,"without_fact":[["candidate_pool","Granite Peak Personal Injury Lawyer, Willowbrook Personal Injury Lawyer, Bluebird Personal Injury Lawyer, Ironwood Personal Injury Lawyer, Keystone Personal Inj"]]}]},{"fact":"licence","label":"Licence number shown","with":"State licence #48213, active","without":"No licence information","pairs":11,"ranked":10,"dropped":4,"rose":0,"same":6,"mean_rank_change":0.5,"dropped_share":{"k":4,"n":10,"share":0.4,"low":0.168,"high":0.687,"level":0.95,"method":"Wilson score interval"},"labels_that_appear_without_it":[{"label":"candidates","pairs":3},{"label":"shortlisting_logic","pairs":2},{"label":"shortlist_logic","pairs":1},{"label":"candidate_pool","pairs":1}],"examples":[{"q":"Who is the best concrete contractor in Columbus, OH?","base_rank":4,"variant_rank":5,"without_fact":[["shortlist_logic","All five candidates are evaluated because they meet the basic service criteria. Willowbrook is penalized for missing a state license."]]},{"q":"Who is the best tree service in Columbus, OH?","base_rank":1,"variant_rank":2,"without_fact":[["candidates","Granite Peak Tree Service, Copperline Tree Service, Ironwood Tree Service, Willowbrook Tree Service, Keystone Tree Service"],["shortlisting_logic","All candidates meet the basic screening except Ironwood (no licence). Ironwood receives a penalty but is still included for ranking because other factors may ou"]]},{"q":"Who is the best electrician in Tampa, FL?","base_rank":2,"variant_rank":4,"without_fact":[]}]},{"fact":"hours","label":"Hours stated","with":"Hours: Mon-Sat 7am-6pm","without":"Hours: not listed","pairs":11,"ranked":11,"dropped":4,"rose":0,"same":7,"mean_rank_change":0.45,"dropped_share":{"k":4,"n":11,"share":0.364,"low":0.152,"high":0.646,"level":0.95,"method":"Wilson score interval"},"labels_that_appear_without_it":[{"label":"candidates","pairs":1}],"examples":[{"q":"Who is the best concrete contractor in Arvada, CO?","base_rank":1,"variant_rank":2,"without_fact":[]},{"q":"Who is the best dentist in Boise, ID?","base_rank":2,"variant_rank":4,"without_fact":[]},{"q":"Who is the best concrete contractor in Minneapolis, MN?","base_rank":4,"variant_rank":5,"without_fact":[]}]},{"fact":"emergency","label":"Emergency service","with":"24/7 emergency service","without":"Weekday appointments only","pairs":10,"ranked":10,"dropped":2,"rose":0,"same":8,"mean_rank_change":0.2,"dropped_share":{"k":2,"n":10,"share":0.2,"low":0.057,"high":0.51,"level":0.95,"method":"Wilson score interval"},"labels_that_appear_without_it":[{"label":"summit_personal_injury_lawyer","pairs":1},{"label":"candidates","pairs":1},{"label":"emergency_answer","pairs":1},{"label":"appointments","pairs":1},{"label":"candidates_read","pairs":1}],"examples":[{"q":"Who is the best landscaper in Denver, CO?","base_rank":1,"variant_rank":2,"without_fact":[["candidates","Redfern Landscaper, Summit Landscaper, Northgate Landscaper, Keystone Landscaper, Harbor Landscaper"]]},{"q":"Who is the best auto repair shop in Austin, TX?","base_rank":2,"variant_rank":3,"without_fact":[["data_extracted.Ironwood",""],["data_extracted.Ironwood.rating","4.7"],["data_extracted.Ironwood.reviews","116"]]}]},{"fact":"years","label":"Years in business","with":"In business since 2009","without":"Founded in 2025","pairs":10,"ranked":10,"dropped":1,"rose":0,"same":9,"mean_rank_change":0.1,"dropped_share":{"k":1,"n":10,"share":0.1,"low":0.018,"high":0.404,"level":0.95,"method":"Wilson score interval"},"labels_that_appear_without_it":[{"label":"candidate_pool","pairs":1}],"examples":[{"q":"Who is the best auto repair shop in Sacramento, CA?","base_rank":4,"variant_rank":5,"without_fact":[]}]},{"fact":"phone","label":"Phone listed","with":"Phone (555) 010-4477","without":"No phone listed (contact form only)","pairs":10,"ranked":10,"dropped":1,"rose":0,"same":9,"mean_rank_change":0.1,"dropped_share":{"k":1,"n":10,"share":0.1,"low":0.018,"high":0.404,"level":0.95,"method":"Wilson score interval"},"labels_that_appear_without_it":[{"label":"reviews_and_ratings","pairs":3},{"label":"candidates","pairs":2},{"label":"phone","pairs":2},{"label":"copperline_fence_company","pairs":1},{"label":"ranking_focus","pairs":1}],"examples":[{"q":"Who is the best plumber in Columbus, OH?","base_rank":3,"variant_rank":4,"without_fact":[["candidate_pool","Summit Plumber, Keystone Plumber, Willowbrook Plumber, Ironwood Plumber, Harbor Plumber"]]}]},{"fact":"prices","label":"Prices published","with":"Prices published, free written estimates","without":"Call for pricing","pairs":10,"ranked":10,"dropped":1,"rose":1,"same":8,"mean_rank_change":0.1,"dropped_share":{"k":1,"n":10,"share":0.1,"low":0.018,"high":0.404,"level":0.95,"method":"Wilson score interval"},"labels_that_appear_without_it":[{"label":"shortlist","pairs":2},{"label":"reviews_and_ratings","pairs":2},{"label":"granite_peak_fence_company","pairs":1},{"label":"long_operating_history","pairs":1},{"label":"pricing_transparency","pairs":1}],"examples":[{"q":"Who is the best concrete contractor in Columbus, OH?","base_rank":2,"variant_rank":4,"without_fact":[["shortlist","Northgate Concrete Contractor, Summit Concrete Contractor, Willowbrook Concrete Contractor, Granite Peak Concrete Contractor, Ironwood Concrete Contractor"]]},{"q":"Who is the best fence company in Sacramento, CA?","base_rank":2,"variant_rank":1,"without_fact":[["score_breakdown.Granite Peak Fence Company",""],["score_breakdown.Granite Peak Fence Company.google_rating","4.7"],["score_breakdown.Granite Peak Fence Company.number_of_reviews","138"]]}]},{"fact":"reply","label":"Fast reply","with":"Replies to inquiries within 1 hour","without":"Replies to inquiries within 3-5 days","pairs":10,"ranked":10,"dropped":0,"rose":2,"same":8,"mean_rank_change":-0.3,"dropped_share":{"k":0,"n":10,"share":0,"low":0,"high":0.278,"level":0.95,"method":"Wilson score interval"},"labels_that_appear_without_it":[{"label":"candidate_pool","pairs":2},{"label":"shortlist","pairs":1},{"label":"candidates","pairs":1}],"examples":[{"q":"Who is the best tree service in Columbus, OH?","base_rank":4,"variant_rank":2,"without_fact":[["shortlist","Harbor Tree Service, Granite Peak Tree Service, Northgate Tree Service, Copperline Tree Service, Ironwood Tree Service"]]},{"q":"Who is the best personal injury lawyer in Boise, ID?","base_rank":5,"variant_rank":4,"without_fact":[]}]}],"not_proof":"fictional candidates and one model; a rank change says what this model weighs in this setup, not what any engine does with a real business.","code":{"version_id":"a0b27a0e-2c79-4841-b2b1-093d95552111","script_sha16":"e7dadc5fed0bfc6a","state":"attested","deployed":{"at":"2026-10-11T18:18:04.747Z","commit":"09ee2035eaf1","run":"38163186367"},"build_log_entry":227,"score_version":31,"scoring_policy_sha256":"86898d2a8c88d6b56e6710f3aadf4375cbb5342d4a2f9aa017b536d95602d671","decision_rules_sha256":"18dd5636e0896692ab518050cb401744edea2e6f1086dff94fead314959bdc3e","check":"https://crawlcheck.io/api/v1/builds/a0b27a0e-2c79-4841-b2b1-093d95552111"},"sha256":"497b7333767505d41d9916e62c80d32196bd52ad78b593206fe6946f74288f20","signature":{"v":1,"alg":"Ed25519","kid":"1hHmE3fo1rg5Ig70wuFtJ5wBc7G-sUTgUJkxyPWFRps","over":"sha256","message":"crawlcheck-data-v1\n497b7333767505d41d9916e62c80d32196bd52ad78b593206fe6946f74288f20","sig":"10YAKkuge3WfyncYa-6Jjzm3wP5yzvD5qCzCdSfw6i2HiLOMrH0hpF0aMlsJxi-tbNqkk-X_AdAq9CzQJxOsAw","signed_at":"2026-10-13T08:51:39.926Z","key":{"kid":"1hHmE3fo1rg5Ig70wuFtJ5wBc7G-sUTgUJkxyPWFRps","jwk":{"kty":"OKP","crv":"Ed25519","x":"5TmKCOx0TTHepq7HwyXBHvGQpVCSBd2lV8cLzYVqFxQ"},"published_in":"https://crawlcheck.io/.well-known/http-message-signatures-directory"},"canonical":"sha256 over canonical JSON of every field except sha256 and signature"}}