14 lines
242 B
Plaintext
14 lines
242 B
Plaintext
|
User-Agent: *
|
||
|
|
|
||
|
|
# Don't allow any bot to crawl tags.
|
||
|
|
Disallow: /tags/
|
||
|
|
Disallow: /tags/*
|
||
|
|
|
||
|
|
# Don't allow bots to crawl through the proxy
|
||
|
|
Disallow: /proxy/*
|
||
|
|
|
||
|
|
{% for user_agent in user_agents %}
|
||
|
|
User-agent: {{user_agent}}
|
||
|
|
Disallow: /
|
||
|
|
{% endfor %}
|