# htts://www.imperva.com/blog/know-your-top-10-bots/
# https://www.keycdn.com/blog/web-crawlers

\.google\.com$
\.googlebot\.com$
\.petalsearch\.com$
\.applebot\.apple\.com$
\.crawl\.yahoo\.net$
\.yandex\.ru$
\.yandex\.com$
\.yandex\.net$
\.baidu\.com$