Over a million developers have joined DZone.

Robots.txt

·
// commented example of robots.txt 


# example robots.txt 
# comments start with "#"

# blocks dissallow directories for useragent
# * means all user agents!
# user agents may contain wildcards
# directory names may not!

User-agent: xyz/1.0
User-agent: ya_robot
Disallow: /special/secret/

User-agent: *
Disallow: /temp/
Disallow: /rework/
Topics:

{{ parent.title || parent.header.title}}

{{ parent.tldr }}

{{ parent.urlSource.name }}