{"success":true,"data":{"engine":"2.0","url":"https://lab2-clean.crawlcheck.io/","domain":"lab2-clean.crawlcheck.io","timestamp":"2026-11-04T05:59:48.547Z","overallScore":80,"uncappedScore":80,"gate":null,"grade":{"letter":"B","label":"Good"},"categories":[{"key":"access","name":"AI Crawler Access","weight":30,"score":99,"passed":9,"total":10},{"key":"technical","name":"Technical SEO","weight":20,"score":100,"passed":10,"total":10},{"key":"content","name":"Content Structure","weight":20,"score":60,"passed":6,"total":10},{"key":"structured","name":"Structured Data","weight":15,"score":63,"passed":2,"total":5},{"key":"trust","name":"Trust & Authority","weight":15,"score":60,"passed":2,"total":3}],"criteria":[{"id":"ai_search_robots","cat":"access","name":"AI search bots allowed in robots.txt","weight":5,"status":"pass","score":100,"evidence":"OAI-SearchBot, Claude-SearchBot and PerplexityBot may fetch this URL.","why":"OAI-SearchBot, Claude-SearchBot and PerplexityBot build the indexes behind ChatGPT search, Claude search and Perplexity answers. Blocking them removes the page from those answers.","fix":"Give each AI search bot its own group with Allow: /, or remove the Disallow that catches them.","tool":{"href":"/tools/robots-txt-validator","label":"robots.txt validator"},"gain":0},{"id":"ai_user_robots","cat":"access","name":"AI assistants allowed in robots.txt","weight":3,"status":"pass","score":100,"evidence":"ChatGPT-User, Claude-User and Perplexity-User are allowed.","why":"ChatGPT-User, Claude-User and Perplexity-User fetch a page when a person asks about it. A block means the assistant says it cannot open your page.","fix":"Allow the user-triggered agents; they fetch single pages on request and add little load.","tool":{"href":"/tools/robots-txt-generator","label":"robots.txt generator"},"gain":0},{"id":"search_engine_robots","cat":"access","name":"Search engines allowed in robots.txt","weight":5,"status":"pass","score":100,"evidence":"Googlebot and Bingbot may fetch this URL.","why":"Googlebot and Bingbot feed Google AI Overviews, AI Mode and Microsoft Copilot as well as classic search.","fix":"Remove the Disallow that applies to Googlebot or Bingbot for this path.","tool":{"href":"/tools/robots-txt-validator","label":"robots.txt validator"},"gain":0},{"id":"training_policy","cat":"access","name":"AI training policy is explicit","weight":1,"status":"warn","score":50,"evidence":"No training crawler is named; all follow the wildcard group.","why":"Blocking or allowing training crawlers does not change AI search visibility, but naming them shows the choice was deliberate rather than inherited from a wildcard rule.","fix":"Add named groups for GPTBot, ClaudeBot, Google-Extended and the other training tokens, set to the policy you want.","tool":{"href":"/blog/training-bots-vs-search-bots","label":"training vs search bots guide"},"gain":0.4},{"id":"firewall","cat":"access","name":"No firewall block for AI crawlers","weight":5,"status":"pass","score":100,"evidence":"GPTBot: 200, OAI-SearchBot: 200. Tested by user agent from our server.","why":"A CDN or WAF can refuse AI crawlers even when robots.txt allows them. In 2026 this is the most common hidden cause of AI invisibility.","fix":"Allow verified AI search bots in your CDN or WAF bot settings, then re-run the check.","tool":{"href":"/tools/ai-crawler-diagnostic","label":"AI crawler diagnostic"},"gain":0},{"id":"noindex","cat":"access","name":"Page is indexable (no noindex)","weight":5,"status":"pass","score":100,"evidence":"No noindex directive.","why":"noindex in the meta robots tag or X-Robots-Tag header keeps the page out of Google, Bing and AI search indexes.","fix":"Remove noindex from pages you want cited.","gain":0},{"id":"nosnippet","cat":"access","name":"Snippets allowed (no nosnippet)","weight":3,"status":"pass","score":100,"evidence":"Snippets are allowed.","why":"nosnippet or max-snippet:0 lets engines index the page but stops them quoting it, which also removes it from AI Overviews text.","fix":"Remove nosnippet, or use data-nosnippet on the specific passages you want hidden.","gain":0},{"id":"server_rendered","cat":"access","name":"Content is in the HTML without JavaScript","weight":5,"status":"pass","score":100,"evidence":"1,128 characters of text are present before any script runs.","why":"Most AI fetchers (GPTBot, ClaudeBot, PerplexityBot) do not run JavaScript. Text that only appears after scripts run is invisible to them.","fix":"Server-render or pre-render the main content (SSR, SSG, or a prerender service).","tool":{"href":"/blog/javascript-rendering-and-ai-crawlers","label":"JavaScript and AI crawlers guide"},"gain":0},{"id":"robots_file","cat":"access","name":"robots.txt is valid and reachable","weight":2,"status":"pass","score":100,"evidence":"Found at https://lab2-clean.crawlcheck.io/robots.txt.","why":"Crawlers fetch robots.txt first. A 5xx makes Google and Semrush stop crawling; an HTML page at that URL is ignored.","fix":"Serve /robots.txt as plain text with a 200 status.","tool":{"href":"/tools/robots-txt-generator","label":"robots.txt generator"},"gain":0},{"id":"llms_txt","cat":"access","name":"llms.txt published","weight":1,"status":"pass","score":100,"evidence":"Found (214 words).","why":"llms.txt is a proposed summary file for AI assistants. No major AI engine has documented reading it, so it is scored as an emerging signal only.","fix":"Publish /llms.txt with a one-paragraph summary and links to your key pages.","tool":{"href":"/tools/llms-txt-generator","label":"llms.txt generator"},"gain":0},{"id":"https","cat":"technical","name":"Served over HTTPS","weight":4,"status":"pass","score":100,"evidence":"The page is served over HTTPS.","why":"HTTPS is a baseline trust and ranking signal for every search engine.","fix":"Install a TLS certificate and serve every page over HTTPS.","gain":0},{"id":"http_redirect","cat":"technical","name":"HTTP redirects to HTTPS","weight":2,"status":"pass","score":100,"evidence":"HTTP 301 to https://lab2-clean.crawlcheck.io/.","why":"If http:// still serves pages, engines can index two copies and split signals between them.","fix":"Add a permanent 301 redirect from http:// to https:// for every URL.","gain":0},{"id":"title","cat":"technical","name":"Title tag present and 15 to 65 characters","weight":4,"status":"pass","score":100,"evidence":"\"Northfield Gutter Co. | Seamless gutters in Northfield\" (54 characters).","why":"The title is the strongest single on-page relevance signal and is often the text an AI answer shows as the citation label.","fix":"Write a unique title of about 50 to 60 characters that states the page topic first.","gain":0},{"id":"meta_description","cat":"technical","name":"Meta description 70 to 170 characters","weight":3,"status":"pass","score":100,"evidence":"125 characters.","why":"Search engines use it for the result snippet and some AI features use it as the page summary.","fix":"Write a 140 to 160 character description that answers the main question the page covers.","tool":{"href":"/blog/ai-overviews-optimization-guide","label":"AI Overviews guide"},"gain":0},{"id":"canonical","cat":"technical","name":"Canonical tag points to this URL","weight":3,"status":"pass","score":100,"evidence":"Self-referencing: https://lab2-clean.crawlcheck.io/.","why":"The canonical tells engines which URL to index and cite when duplicates exist. A canonical to another URL tells them not to use this one.","fix":"Add <link rel=\"canonical\"> with this page's own absolute URL.","gain":0},{"id":"sitemap","cat":"technical","name":"XML sitemap found","weight":3,"status":"pass","score":100,"evidence":"Sitemap index with 1 child sitemaps at https://lab2-clean.crawlcheck.io/sitemap.xml.","why":"Sitemaps help crawlers discover pages and notice updates. Bing states that it uses lastmod to prioritise recrawls for Copilot.","fix":"Publish an XML sitemap and reference it with a Sitemap: line in robots.txt.","tool":{"href":"/blog/sitemap-xml-for-ai-discoverability","label":"sitemap guide"},"gain":0},{"id":"lang","cat":"technical","name":"Language declared","weight":2,"status":"pass","score":100,"evidence":"lang=\"en\".","why":"The html lang attribute tells engines which language the answer is in, which matters for multilingual AI answers.","fix":"Add lang to the <html> element, for example lang=\"en\".","gain":0},{"id":"hreflang","cat":"technical","name":"hreflang for alternate languages","weight":1,"status":"na","score":0,"evidence":"No hreflang tags. Not needed for single-language sites.","why":"Only relevant for sites with several language or region versions.","fix":"Add hreflang links for each language version, including x-default.","gain":0},{"id":"viewport","cat":"technical","name":"Mobile viewport set","weight":2,"status":"pass","score":100,"evidence":"Viewport meta tag present.","why":"Google indexes the mobile version first; a page without a viewport renders as a desktop page on phones.","fix":"Add <meta name=\"viewport\" content=\"width=device-width, initial-scale=1\">.","gain":0},{"id":"response_time","cat":"technical","name":"Server responds quickly","weight":2,"status":"pass","score":100,"evidence":"24 ms to first byte (measured from our server).","why":"Crawlers have fetch timeouts and crawl budgets; slow responses mean fewer pages fetched and refreshed.","fix":"Cache HTML at the edge or reduce server work before the first byte.","gain":0},{"id":"page_weight","cat":"technical","name":"HTML size under 1 MB","weight":1,"status":"pass","score":100,"evidence":"4 KB of HTML.","why":"Very large HTML documents can be truncated by crawlers before the main content.","fix":"Move inline data and scripts out of the HTML document.","gain":0},{"id":"h1","cat":"content","name":"Exactly one H1","weight":3,"status":"pass","score":100,"evidence":"\"Seamless gutters, fitted right the first time\".","why":"The H1 states what the page is about; engines use it to label the passage they extract.","fix":"Use one H1 that matches the page topic; turn other H1s into H2s.","gain":0},{"id":"subheadings","cat":"content","name":"Sections under descriptive H2s","weight":3,"status":"warn","score":50,"evidence":"Only 1 H2.","why":"AI engines extract passages section by section. Clear H2s let them find the part that answers a question.","fix":"Break the content into sections of 150 to 300 words, each under an H2 that names its topic.","gain":1.3},{"id":"heading_order","cat":"content","name":"Heading levels in order","weight":1,"status":"pass","score":100,"evidence":"No skipped heading levels.","why":"Skipped levels (H2 to H4) make the outline harder for parsers and screen readers to follow.","fix":"Do not skip heading levels.","gain":0},{"id":"content_depth","cat":"content","name":"Enough main content to cite","weight":4,"status":"fail","score":0,"evidence":"144 words of main content.","why":"Engines cite pages that answer a question fully. A few sentences rarely win a citation over a thorough source.","fix":"Expand the main content with the details, examples and data a reader would ask for next.","gain":3.3},{"id":"question_headings","cat":"content","name":"Questions answered in headings","weight":3,"status":"fail","score":0,"evidence":"No headings phrased as questions.","why":"People ask AI assistants questions. Headings phrased as those questions, followed by a direct answer, map cleanly onto prompts.","fix":"Phrase two or more headings as the questions your audience asks, and answer each in the first sentence below it.","tool":{"href":"/blog/answer-first-content-structure","label":"answer-first structure guide"},"gain":2.5},{"id":"lists_tables","cat":"content","name":"Lists or tables for scannable facts","weight":2,"status":"warn","score":50,"evidence":"1 list, 0 tables.","why":"Steps, comparisons and specs in lists and tables are easy for engines to lift into an answer.","fix":"Put steps in ordered lists and comparisons in tables with header cells.","gain":0.8},{"id":"readability","cat":"content","name":"Sentences and paragraphs are short","weight":2,"status":"pass","score":100,"evidence":"Average sentence 16 words; no paragraph over 150 words.","why":"Short, self-contained sentences are easier to quote accurately.","fix":"Aim for sentences under 22 words and paragraphs under 120 words.","gain":0},{"id":"semantic_html","cat":"content","name":"Main content marked with <main> or <article>","weight":2,"status":"pass","score":100,"evidence":"Landmarks: header, nav, main, footer.","why":"Landmarks let parsers separate the content from navigation and footers, so the right text is extracted.","fix":"Wrap the primary content in <main>, and posts in <article>.","gain":0},{"id":"image_alt","cat":"content","name":"Images have alt attributes","weight":2,"status":"pass","score":100,"evidence":"3 of 3 images have alt.","why":"Alt text is the only description of an image a text crawler receives.","fix":"Add descriptive alt text to content images and alt=\"\" to decorative ones.","gain":0},{"id":"internal_links","cat":"content","name":"Internal links to related pages","weight":2,"status":"pass","score":100,"evidence":"29 internal links.","why":"Internal links help crawlers discover related pages and understand topic clusters.","fix":"Link to the related guides, products and hub pages with descriptive anchor text.","gain":0},{"id":"schema_present","cat":"structured","name":"Valid JSON-LD structured data","weight":4,"status":"pass","score":100,"evidence":"Types: LocalBusiness, PostalAddress.","why":"Structured data states facts about the page in a form engines do not have to guess. Pages cited by AI Mode and ChatGPT carry it far more often than not.","fix":"Add JSON-LD for the page type (Organization, Article, Product, FAQPage) in the <head>.","tool":{"href":"/tools/schema-generator","label":"schema markup generator"},"gain":0},{"id":"schema_entity","cat":"structured","name":"Organization or Person entity","weight":3,"status":"pass","score":100,"evidence":"Entity present with 1 sameAs link.","why":"An Organization or Person node tells engines who publishes the content, which links the page to a known entity.","fix":"Add an Organization node with name, url, logo and sameAs links to your official profiles.","tool":{"href":"/tools/schema-generator","label":"schema markup generator"},"gain":0},{"id":"schema_content","cat":"structured","name":"Content-type schema (Article, Product, FAQ...)","weight":3,"status":"warn","score":30,"evidence":"Only site-level types (WebSite, WebPage, BreadcrumbList) or none.","why":"A type that describes the page itself (Article, Product, FAQPage, HowTo, Service) gives engines the headline, dates and answers directly.","fix":"Add the schema type that matches the page: Article for posts, Product for products, FAQPage for visible Q&A.","tool":{"href":"/tools/schema-generator","label":"schema markup generator"},"gain":2.4},{"id":"schema_dates","cat":"structured","name":"Dates in schema (published and modified)","weight":2,"status":"na","score":0,"evidence":"Not an article page, so publish dates are optional.","why":"AI engines favour fresh sources for time-sensitive questions and read dateModified to judge freshness.","fix":"Add datePublished and dateModified to the Article or WebPage node, and update dateModified when content changes.","gain":0},{"id":"breadcrumbs","cat":"structured","name":"BreadcrumbList schema","weight":1,"status":"warn","score":30,"evidence":"No BreadcrumbList.","why":"Breadcrumbs describe where the page sits in the site, which helps engines understand topical hierarchy.","fix":"Add BreadcrumbList JSON-LD that mirrors the visible breadcrumb trail.","gain":0.8},{"id":"open_graph","cat":"structured","name":"Open Graph title, description and image","weight":2,"status":"fail","score":0,"evidence":"No Open Graph tags.","why":"Open Graph tags control how the page appears when shared in chat apps and assistants that render link previews.","fix":"Add og:title, og:description and og:image.","gain":2.3},{"id":"author","cat":"trust","name":"Author shown or declared","weight":3,"status":"na","score":0,"evidence":"Not an article page, so a byline is not expected. Articles on the site should carry one.","why":"A named author is a core E-E-A-T signal, and engines are more willing to cite content with accountable authorship.","fix":"Show a byline linked to an author page, and add author to the Article schema.","tool":{"href":"/blog/author-bylines-credentials-ai-trust","label":"author bylines guide"},"gain":0},{"id":"visible_date","cat":"trust","name":"Visible publish or update date","weight":2,"status":"na","score":0,"evidence":"Not an article page, so a visible date is optional.","why":"A visible date tells readers and engines how current the information is.","fix":"Show the publish and last-updated dates in a <time datetime> element.","gain":0},{"id":"outbound_sources","cat":"trust","name":"Links to supporting sources","weight":2,"status":"fail","score":0,"evidence":"No links to external sources.","why":"Citing sources shows the claims are checkable, which is part of how engines judge trustworthiness.","fix":"Link to the primary sources behind your data and claims.","gain":6},{"id":"about_contact","cat":"trust","name":"About or contact page linked","weight":2,"status":"pass","score":100,"evidence":"Link to an about or contact page found.","why":"A reachable About or Contact page shows there is an accountable organisation behind the site.","fix":"Link to your About and Contact pages from the header or footer.","gain":0},{"id":"mixed_content","cat":"trust","name":"No insecure (http) resources","weight":1,"status":"pass","score":100,"evidence":"All resources load over HTTPS.","why":"Mixed content triggers browser warnings and weakens the page's security signals.","fix":"Load every image, script and stylesheet over HTTPS.","gain":0}],"recommendations":[{"id":"content_depth","title":"Enough main content to cite","fix":"Expand the main content with the details, examples and data a reader would ask for next.","why":"Engines cite pages that answer a question fully. A few sentences rarely win a citation over a thorough source.","category":"Content Structure","priority":"high","gain":3.3,"evidence":"144 words of main content."},{"id":"outbound_sources","title":"Links to supporting sources","fix":"Link to the primary sources behind your data and claims.","why":"Citing sources shows the claims are checkable, which is part of how engines judge trustworthiness.","category":"Trust & Authority","priority":"medium","gain":6,"evidence":"No links to external sources."},{"id":"question_headings","title":"Questions answered in headings","fix":"Phrase two or more headings as the questions your audience asks, and answer each in the first sentence below it.","why":"People ask AI assistants questions. Headings phrased as those questions, followed by a direct answer, map cleanly onto prompts.","category":"Content Structure","priority":"medium","gain":2.5,"evidence":"No headings phrased as questions.","tool":{"href":"/blog/answer-first-content-structure","label":"answer-first structure guide"}},{"id":"schema_content","title":"Content-type schema (Article, Product, FAQ...)","fix":"Add the schema type that matches the page: Article for posts, Product for products, FAQPage for visible Q&A.","why":"A type that describes the page itself (Article, Product, FAQPage, HowTo, Service) gives engines the headline, dates and answers directly.","category":"Structured Data","priority":"medium","gain":2.4,"evidence":"Only site-level types (WebSite, WebPage, BreadcrumbList) or none.","tool":{"href":"/tools/schema-generator","label":"schema markup generator"}},{"id":"open_graph","title":"Open Graph title, description and image","fix":"Add og:title, og:description and og:image.","why":"Open Graph tags control how the page appears when shared in chat apps and assistants that render link previews.","category":"Structured Data","priority":"medium","gain":2.3,"evidence":"No Open Graph tags."},{"id":"subheadings","title":"Sections under descriptive H2s","fix":"Break the content into sections of 150 to 300 words, each under an H2 that names its topic.","why":"AI engines extract passages section by section. Clear H2s let them find the part that answers a question.","category":"Content Structure","priority":"medium","gain":1.3,"evidence":"Only 1 H2."},{"id":"lists_tables","title":"Lists or tables for scannable facts","fix":"Put steps in ordered lists and comparisons in tables with header cells.","why":"Steps, comparisons and specs in lists and tables are easy for engines to lift into an answer.","category":"Content Structure","priority":"low","gain":0.8,"evidence":"1 list, 0 tables."},{"id":"breadcrumbs","title":"BreadcrumbList schema","fix":"Add BreadcrumbList JSON-LD that mirrors the visible breadcrumb trail.","why":"Breadcrumbs describe where the page sits in the site, which helps engines understand topical hierarchy.","category":"Structured Data","priority":"low","gain":0.8,"evidence":"No BreadcrumbList."},{"id":"training_policy","title":"AI training policy is explicit","fix":"Add named groups for GPTBot, ClaudeBot, Google-Extended and the other training tokens, set to the policy you want.","why":"Blocking or allowing training crawlers does not change AI search visibility, but naming them shows the choice was deliberate rather than inherited from a wildcard rule.","category":"AI Crawler Access","priority":"low","gain":0.4,"evidence":"No training crawler is named; all follow the wildcard group.","tool":{"href":"/blog/training-bots-vs-search-bots","label":"training vs search bots guide"}}],"crawlers":[{"token":"OAI-SearchBot","operator":"OpenAI","tier":"AI search","allowed":true,"matched":"*","rule":"Allow: / (line 3)"},{"token":"Claude-SearchBot","operator":"Anthropic","tier":"AI search","allowed":true,"matched":"*","rule":"Allow: / (line 3)"},{"token":"PerplexityBot","operator":"Perplexity","tier":"AI search","allowed":true,"matched":"*","rule":"Allow: / (line 3)"},{"token":"ChatGPT-User","operator":"OpenAI","tier":"AI assistant","allowed":true,"matched":"*","rule":"Allow: / (line 3)"},{"token":"Claude-User","operator":"Anthropic","tier":"AI assistant","allowed":true,"matched":"*","rule":"Allow: / (line 3)"},{"token":"Perplexity-User","operator":"Perplexity","tier":"AI assistant","allowed":true,"matched":"*","rule":"Allow: / (line 3)"},{"token":"GPTBot","operator":"OpenAI","tier":"AI training","allowed":true,"matched":"*","rule":"Allow: / (line 3)"},{"token":"ClaudeBot","operator":"Anthropic","tier":"AI training","allowed":true,"matched":"*","rule":"Allow: / (line 3)"},{"token":"Google-Extended","operator":"Google","tier":"AI training","allowed":true,"matched":"*","rule":"Allow: / (line 3)"},{"token":"Applebot-Extended","operator":"Apple","tier":"AI training","allowed":true,"matched":"*","rule":"Allow: / (line 3)"},{"token":"Meta-ExternalAgent","operator":"Meta","tier":"AI training","allowed":true,"matched":"*","rule":"Allow: / (line 3)"},{"token":"CCBot","operator":"Common Crawl","tier":"AI training","allowed":true,"matched":"*","rule":"Allow: / (line 3)"},{"token":"Googlebot","operator":"Google","tier":"Search engine","allowed":true,"matched":"*","rule":"Allow: / (line 3)"},{"token":"bingbot","operator":"Microsoft","tier":"Search engine","allowed":true,"matched":"*","rule":"Allow: / (line 3)"},{"token":"Applebot","operator":"Apple","tier":"Search engine","allowed":true,"matched":"*","rule":"Allow: / (line 3)"}],"summary":{"passed":29,"warnings":5,"failed":4,"notApplicable":4,"total":42},"fetch":{"status":200,"ttfbMs":24,"finalUrl":"https://lab2-clean.crawlcheck.io/","htmlKb":4,"robotsTxt":{"found":true,"status":200,"note":"Found at https://lab2-clean.crawlcheck.io/robots.txt"},"sitemap":{"found":true,"url":"https://lab2-clean.crawlcheck.io/sitemap.xml","urlCount":1,"note":"Sitemap index with 1 child sitemaps"},"llmsTxt":{"found":true,"note":"214 words"},"llmsFullTxt":{"found":false,"note":"HTTP 404"},"probes":[{"label":"GPTBot","status":200,"blocked":false},{"label":"OAI-SearchBot","status":200,"blocked":false}]},"page":{"title":"Northfield Gutter Co. | Seamless gutters in Northfield","metaDescription":"Seamless gutters, gutter guards and cleaning in Northfield and seven nearby towns. Free fixed quotes, ten-year leak warranty.","lang":"en","canonical":"https://lab2-clean.crawlcheck.io/","wordCount":183,"mainWordCount":144,"h1":"Seamless gutters, fitted right the first time","h2":1,"h3":0,"images":3,"imagesWithAlt":3,"internalLinks":29,"externalLinks":0,"schemaTypes":["LocalBusiness","PostalAddress"]}}}