Sitemap: https://gwern.net/sitemap.xml

User-agent: ia_archiver
Disallow: /
Allow: /modafinil
Allow: /dnm-arrest

User-agent: *
Disallow: /fulltext
Disallow: /lorem
# Hide copies:
# Duplicate content is bad for SEO (and clutters search results), so no Markdown sources, WWW archives, metadata snippets, or link-bibliography compilations.
Disallow: /*.md
Disallow: /*.md.html
# disallow syntax-highlighted versions of source code as duplicates:
Disallow: /static/*.*.html
Disallow: /metadata/*
Disallow: /metadata/annotation/backlink/*
Disallow: /metadata/annotation/similar/*
Disallow: /metadata/annotation/link-bibliography/*
Disallow: /confidential/*
Disallow: /private/*
Disallow: /secret/*
Disallow: /doc/www/*
# # I disallow doc/*/index pages because those keep cluttering up Google Scholar
# Disallow: /doc/*/index
# Allow: /doc/rotten.com/*