# Small robots.txt # More information about this file can be found at # In case your drupal site is in a subdirectory of your web root (e.g. /drupal) # add the name of this directory before the / (slash) below # example: Disallow: /drupal/aggregator # to stop a polite robot indexing an example dir # add a line like: user-agent: polite-bot # and: Disallow: /example-dir/ # Paths (clean URLs) User-agent: * Crawl-Delay: 10 Disallow: /includes Disallow: /logs Disallow: /misc Disallow: /profiles Disallow: /scripts Disallow: /files Disallow: /sites Disallow: /themes Disallow: /tmp_upload # Paths (no clean URLs) User-agent: * Crawl-Delay: 10 Disallow: /?q=aggregator Disallow: /?q=tracker Disallow: /?q=comment/reply Disallow: /?q=node/add Disallow: /?q=user Disallow: /?q=files Disallow: /?q=search Disallow: /?q=book/print