Site Architecture Skill
For technical website structure that enables discovery by search engines AND AI crawlers (GPTBot, ClaudeBot, PerplexityBot).
Philosophy
Content is king. Architecture is the kingdom.
Great content buried in poor architecture won't be discovered. This skill covers the technical foundation that makes your content findable by:
- Google, Bing (traditional search)
- GPTBot (ChatGPT), ClaudeBot, PerplexityBot (AI assistants)
- Social platforms (Open Graph, Twitter Cards)
robots.txt
Basic Template
txt
# robots.txt
# Allow all crawlers by default
User-agent: *
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /private/
Disallow: /_next/
Disallow: /cdn-cgi/
# Sitemap location
Sitemap: https://yoursite.com/sitemap.xml
# Crawl delay (optional - be careful, not all bots respect this)
# Crawl-delay: 1
AI Bot Configuration
txt
# robots.txt with AI bot rules
# === SEARCH ENGINES ===
User-agent: Googlebot
Allow: /
User-agent: Bingbot
Allow: /
# === AI ASSISTANTS (Allow for discovery) ===
User-agent: GPTBot
Allow: /
User-agent: ChatGPT-User
Allow: /
User-agent: Claude-Web
Allow: /
User-agent: ClaudeBot
Allow: /
User-agent: PerplexityBot
Allow: /
User-agent: Amazonbot
Allow: /
User-agent: anthropic-ai
Allow: /
User-agent: Google-Extended
Allow: /
# === BLOCK AI TRAINING (Optional - block training, allow chat) ===
# Uncomment these if you want to be cited but not used for training
# User-agent: CCBot
# Disallow: /
# User-agent: GPTBot
# Disallow: / # Blocks both chat and training
# === BLOCK SCRAPERS ===
User-agent: AhrefsBot
Disallow: /
User-agent: SemrushBot
Disallow: /
User-agent: MJ12bot
Disallow: /
# === DEFAULT ===
User-agent: *
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /auth/
Disallow: /private/
Disallow: /*.json$
Disallow: /*?*
Sitemap: https://yoursite.com/sitemap.xml
Next.js robots.txt
typescript
// app/robots.ts
import { MetadataRoute } from 'next';
export default function robots(): MetadataRoute.Robots {
const baseUrl = process.env.NEXT_PUBLIC_URL || 'https://yoursite.com';
return {
rules: [
{
userAgent: '*',
allow: '/',
disallow: ['/api/', '/admin/', '/private/', '/_next/'],
},
{
userAgent: 'GPTBot',
allow: '/',
},
{
userAgent: 'ClaudeBot',
allow: '/',
},
{
userAgent: 'PerplexityBot',
allow: '/',
},
],
sitemap: `${baseUrl}/sitemap.xml`,
};
}
Sitemap
XML Sitemap Template
xml
<?xml version="1.0" encoding="UTF-8"?>
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"
xmlns:news="http://www.google.com/schemas/sitemap-news/0.9"
xmlns:image="http://www.google.com/schemas/sitemap-image/1.1">
<url>
<loc>https://yoursite.com/</loc>
<lastmod>2025-01-15</lastmod>
<changefreq>weekly</changefreq>
<priority>1.0</priority>
</url>
<url>
<loc>https://yoursite.com/pricing</loc>
<lastmod>2025-01-10</lastmod>
<changefreq>monthly</changefreq>
<priority>0.9</priority>
</url>
<url>
<loc>https://yoursite.com/blog/article-slug</loc>
<lastmod>2025-01-12</lastmod>
<changefreq>monthly</changefreq>
<priority>0.8</priority>
<image:image>
<image:loc>https://yoursite.com/images/article-image.jpg</image:loc>
</image:image>
</url>
</urlset>
Next.js Dynamic Sitemap
typescript
// app/sitemap.ts
import { MetadataRoute } from 'next';
export default async function sitemap(): Promise<MetadataRoute.Sitemap> {
const baseUrl = process.env.NEXT_PUBLIC_URL || 'https://yoursite.com';
// Static pages
const staticPages = [
{ url: '/', priority: 1.0, changeFrequency: 'weekly' as const },
{ url: '/pricing', priority: 0.9, changeFrequency: 'monthly' as const },
{ url: '/about', priority: 0.8, changeFrequency: 'monthly' as const },
{ url: '/contact', priority: 0.7, changeFrequency: 'yearly' as const },
];
// Dynamic pages (e.g., blog posts)
const posts = await getBlogPosts(); // Your data fetching function
const blogPages = posts.map((post) => ({
url: `/blog/${post.slug}`,
lastModified: new Date(post.updatedAt),
changeFrequency: 'monthly' as const,
priority: 0.8,
}));
return [
...staticPages.map((page) => ({
url: `${baseUrl}${page.url}`,
lastModified: new Date(),
changeFrequency: page.changeFrequency,
priority: page.priority,
})),
...blogPages.map((page) => ({
url: `${baseUrl}${page.url}`,
lastModified: page.lastModified,
changeFrequency: page.changeFrequency,
priority: page.priority,
})),
];
}
Sitemap Index (Large Sites)
xml
<?xml version="1.0" encoding="UTF-8"?>
<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
<sitemap>
<loc>https://yoursite.com/sitemap-pages.xml</loc>
<lastmod>2025-01-15</lastmod>
</sitemap>
<sitemap>
<loc>https://yoursite.com/sitemap-blog.xml</loc>
<lastmod>2025-01-14</lastmod>
</sitemap>
<sitemap>
<loc>https://yoursite.com/sitemap-products.xml</loc>
<lastmod>2025-01-13</lastmod>
</sitemap>
</sitemapindex>
Meta Tags
Essential Meta Tags
html
<head>
<!-- Basic -->
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Page Title | Brand Name</title>
<meta name="description" content="Compelling 150-160 character description with keywords and CTA.">
<!-- Canonical (prevent duplicate content) -->
<link rel="canonical" href="https://yoursite.com/current-page">
<!-- Language -->
<html lang="en">
<meta name="language" content="English">
<!-- Robots -->
<meta name="robots" content="index, follow">
<meta name="googlebot" content="index, follow">
<!-- Author -->
<meta name="author" content="Author Name">
<!-- Favicon -->
<link rel="icon" href="/favicon.ico" sizes="any">
<link rel="icon" href="/icon.svg" type="image/svg+xml">
<link rel="apple-touch-icon" href="/apple-touch-icon.png">
<link rel="manifest" href="/manifest.webmanifest">
</head>
Open Graph (Social Sharing)
html
<!-- Open Graph / Facebook -->
<meta property="og:type" content="website">
<meta property="og:url" content="https://yoursite.com/page">
<meta property="og:title" content="Page Title - Brand">
<meta property="og:description" content="Description for social sharing (can be longer).">
<meta property="og:image" content="https://yoursite.com/og-image.jpg">
<meta property="og:image:width" content="1200">
<meta property="og:image:height" content="630">
<meta property="og:site_name" content="Brand Name">
<meta property="og:locale" content="en_US">
<!-- Article-specific (for blog posts) -->
<meta property="og:type" content="article">
<meta property="article:published_time" content="2025-01-15T08:00:00Z">
<meta property="article:modified_time" content="2025-01-20T10:00:00Z">
<meta property="article:author" content="https://yoursite.com/team/author">
<meta property="article:section" content="Technology">
<meta property="article:tag" content="AI, SEO, Content">
Twitter Cards
html
<!-- Twitter -->
<meta name="twitter:card" content="summary_large_image">
<meta name="twitter:site" content="@yourbrand">
<meta name="twitter:creator" content="@authorhandle">
<meta name="twitter:title" content="Page Title">
<meta name="twitter:description" content="Description for Twitter (max 200 chars).">
<meta name="twitter:image" content="https://yoursite.com/twitter-image.jpg">
Next.js Metadata
typescript
// app/layout.tsx
import { Metadata } from 'next';
export const metadata: Metadata = {
metadataBase: new URL('https://yoursite.com'),
title: {
default: 'Brand Name',
template: '%s | Brand Name',
},
description: 'Your default site description.',
keywords: ['keyword1', 'keyword2', 'keyword3'],
authors: [{ name: 'Brand Name', url: 'https://yoursite.com' }],
creator: 'Brand Name',
publisher: 'Brand Name',
robots: {
index: true,
follow: true,
googleBot: {
index: true,
follow: true,
'max-video-preview': -1,
'max-image-preview': 'large',
'max-snippet': -1,
},
},
openGraph: {
type: 'website',
locale: 'en_US',
url: 'https://yoursite.com',
siteName: 'Brand Name',
title: 'Brand Name',
description: 'Your site description.',
images: [
{
url: '/og-image.jpg',
width: 1200,
height: 630,
alt: 'Brand Name',
},
],
},
twitter: {
card: 'summary_large_image',
site: '@yourbrand',
creator: '@yourbrand',
},
verification: {
google: 'google-verification-code',
yandex: 'yandex-verification-code',
},
};
// app/blog/[slug]/page.tsx
export async function generateMetadata({ params }): Promise<Metadata> {
const post = await getPost(params.slug);
return {
title: post.title,
description: post.excerpt,
openGraph: {
title: post.title,
description: post.excerpt,
type: 'article',
publishedTime: post.publishedAt,
modifiedTime: post.updatedAt,
authors: [post.author.name],
images: [post.coverImage],
},
};
}
URL Structure
Best Practices
markdown
✅ GOOD URLs:
/blog/ai-seo-best-practices
/products/pro-plan
/pricing
/about/team
❌ BAD URLs:
/blog?id=123
/p/12345
/index.php?page=about
/Products/Pro_Plan (inconsistent casing)
URL Guidelines
| Rule | Example | |
|---|
| Lowercase only | /blog/my-post not /Blog/My-Post | |
| Hyphens not underscores | /my-page not /my_page | |
| No trailing slashes | /about not /about/ | |
| Descriptive slugs | /pricing not /p | |
| No query params for content | /blog/post-title not /blog?id=123 | |
| Max 3-4 levels deep | /blog/category/post | |
Redirect Configuration
typescript
// next.config.js
module.exports = {
async redirects() {
return [
// Redirect old URLs to new
{
source: '/old-page',
destination: '/new-page',
permanent: true, // 301 redirect
},
// Redirect with wildcard
{
source: '/blog/old/:slug',
destination: '/articles/:slug',
permanent: true,
},
// Trailing slash redirect
{
source: '/:path+/',
destination: '/:path+',
permanent: true,
},
];
},
};
Canonical URLs
Implementation
html
<!-- Always include canonical, even for primary URL -->
<link rel="canonical" href="https://yoursite.com/current-page">
When to Use
markdown
✅ USE CANONICAL:
-Every page (even if only version exists)
-Paginated content (point to page 1 or use rel=prev/next)
-URL parameters that don't change content (?utm_source=...)
-HTTP vs HTTPS (canonical to HTTPS)
-www vs non-www (pick one, canonical to it)
Example: /products?sort=price should canonical to /products
Next.js Canonical
typescript
// Automatic in metadata
export const metadata: Metadata = {
alternates: {
canonical: '/current-page',
},
};
Security Headers
Essential Headers
typescript
// next.config.js
const securityHeaders = [
{
key: 'X-DNS-Prefetch-Control',
value: 'on',
},
{
key: 'Strict-Transport-Security',
value: 'max-age=63072000; includeSubDomains; preload',
},
{
key: 'X-Frame-Options',
value: 'SAMEORIGIN',
},
{
key: 'X-Content-Type-Options',
value: 'nosniff',
},
{
key: 'Referrer-Policy',
value: 'strict-origin-when-cross-origin',
},
{
key: 'Permissions-Policy',
value: 'camera=(), microphone=(), geolocation=()',
},
];
module.exports = {
async headers() {
return [
{
source: '/:path*',
headers: securityHeaders,
},
];
},
};
Core Web Vitals
Target Metrics