sitemap.xml 与 robots.txt 配置与模板

<?xml version="1.0" encoding="UTF-8"?>
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
  <url>
    <loc>https://example.com/page1.html</loc>
    <lastmod>2024-01-01</lastmod>
    <changefreq>weekly</changefreq>
    <priority>0.8</priority>
  </url>
  <url>
    <loc>https://example.com/page2.html</loc>
    <lastmod>2024-01-01</lastmod>
    <changefreq>monthly</changefreq>
    <priority>0.5</priority>
  </url>
</urlset>

# 允许所有爬虫访问所有内容
User-agent: *
Allow: /

# 阻止特定爬虫访问特定目录
User-agent: BadBot
Disallow: /

# 阻止访问特定目录
User-agent: *
Disallow: /admin/
Disallow: /private/
Disallow: /search

# 指定 sitemap 位置
Sitemap: https://example.com/sitemap.xml

# config/_default/config.toml
[sitemap]
  changefreq = "weekly"
  priority = 0.5
  filename = "sitemap.xml"
  baseURL = "https://example.com"

---
title: "页面标题"
# 从 sitemap 中排除此页面
sitemap:
  disable: true
---

---
title: "重要页面"
# 自定义此页面的 sitemap 设置
sitemap:
  priority: 1.0
  changefreq: "daily"
  lastmod: 2024-01-01
---

<!-- 自定义 sitemap 模板 -->
{{ printf "<?xml version=\"1.0\" encoding=\"utf-8\" standalone=\"yes\" ?>" | safeHTML }}
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"
        xmlns:xhtml="http://www.w3.org/1999/xhtml">
  {{ range where .Site.RegularPages ".Params.sitemap.disable" "!=" true }}
  <url>
    <loc>{{ .Permalink }}</loc>
    {{ if .Lastmod }}
    <lastmod>{{ safeHTML (printf "%.20s" .Lastmod.Format "2006-01-02T15:04:05-07:00") }}</lastmod>
    {{ else if .Date }}
    <lastmod>{{ safeHTML (printf "%.20s" .Date.Format "2006-01-02T15:04:05-07:00") }}</lastmod>
    {{ end }}
    {{ with .Params.sitemap }}
      {{ with .changefreq }}<changefreq>{{ . }}</changefreq>{{ end }}
      {{ with .priority }}<priority>{{ . }}</priority>{{ end }}
    {{ else }}
      {{ with $.Site.Params.sitemap }}
        {{ with .changefreq }}<changefreq>{{ . }}</changefreq>{{ end }}
        {{ with .priority }}<priority>{{ . }}</priority>{{ end }}
      {{ end }}
    {{ end }}
  </url>
  {{ end }}
</urlset>

# 允许所有爬虫访问所有内容
User-agent: *
Allow: /

# 阻止特定爬虫访问特定目录
User-agent: BadBot
Disallow: /

# 阻止访问特定目录
User-agent: *
Disallow: /admin/
Disallow: /private/
Disallow: /search

# 指定 sitemap 位置
Sitemap: https://example.com/sitemap.xml

# static/robots.txt
User-agent: *
Allow: /*

<!-- layouts/robots.txt -->
User-agent: *
{{ range .Site.RegularPages }}
Allow: {{ .RelPermalink }}
{{ end }}

Sitemap: {{ .Site.BaseURL }}/sitemap.xml

# 生产环境 robots.txt
User-agent: *
Allow: /

Sitemap: https://jimmysong.io/sitemap.xml

# 开发环境 robots.txt
User-agent: *
Disallow: /

# 阻止所有爬虫访问开发环境

<!-- 允许索引和跟踪链接 -->
<meta name="robots" content="index, follow">

<!-- 阻止索引但允许跟踪链接 -->
<meta name="robots" content="noindex, follow">

<!-- 阻止索引和跟踪链接 -->
<meta name="robots" content="noindex, nofollow">

<!-- 允许索引但阻止片段显示 -->
<meta name="robots" content="index, follow, max-snippet:0">

<!-- Robots meta -->
{{ $defaultRobots := "index, follow, max-snippet:-1, max-image-preview:large, max-video-preview:-1" }}
{{ $siteRobots := default $defaultRobots .Site.Params.robots }}
{{ $robots := cond (or .Params.noindex .Site.Params.siteWideNoIndex) "noindex, nofollow" $siteRobots }}
<meta name="robots" content="{{ $robots }}">

---
title: "公开页面"
# 默认行为：允许索引和跟踪
---

---
title: "私有页面"
# 阻止索引和跟踪
noindex: true
---

---
title: "敏感内容"
# 阻止片段显示
robots: "noindex, nofollow, max-snippet:0, noarchive"
---

<!-- 根据内容类型设置更新频率 -->
{{ $changefreq := cond
  (eq .Section "blog") "weekly"
  (eq .Section "news") "daily"
  (eq .Section "docs") "monthly"
  "yearly" }}
<changefreq>{{ $changefreq }}</changefreq>

<!-- sitemap 索引文件 -->
<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
  <sitemap>
    <loc>https://example.com/sitemap-blog.xml</loc>
    <lastmod>2024-01-01</lastmod>
  </sitemap>
  <sitemap>
    <loc>https://example.com/sitemap-docs.xml</loc>
    <lastmod>2024-01-01</lastmod>
  </sitemap>
</sitemapindex>

# 检查 sitemap 文件
curl -s "https://example.com/sitemap.xml" | head -20

# 验证 robots.txt 语法
curl -s "https://example.com/robots.txt"

# 检查页面 robots meta 标签
curl -s "https://example.com/page.html" | grep -i "robots"

# 验证 sitemap 中的 URL 是否可访问
curl -I "https://example.com/some-page/"

# robots.txt 性能优化配置
User-agent: *
Crawl-delay: 1
Allow: /

# 允许重要的资源文件
Allow: /*.css$
Allow: /*.js$
Allow: /*.png$
Allow: /*.jpg$
Allow: /*.webp$

Sitemap: https://example.com/sitemap.xml