User-agent: *
Disallow: /admin/
Disallow: /tasks/
Disallow: /core/
Disallow: /config/
Disallow: /sites/cfd/includes/themes/cfd/includes/display_objects/custom/transitinfostructure/
Disallow: /tools-resources/nova-transit-schedules/schedule-change-notification/
Disallow: /tools-resources/nova-point-to-point-schedules/
Disallow: /routes-schedules/schedules/point-to-point-schedules/
Disallow: /routes-schedules/schedules/schedule-change-notification/
Crawl-Delay: 5


User-agent: MegaIndex.ru
Disallow: /

User-agent: MegaIndex.ru/2.0
Disallow: /

User-agent: cludo.com
Disallow: /

User-agent: python-requests
Disallow: /

User-agent: Amazonbot
Disallow: /routes-schedules/schedules/point-to-point-schedules/
Disallow: /tools-resources/nova-point-to-point-schedules/

User-agent: dotbot
Disallow: /