User-agent: *
Disallow: *utm
Disallow: *openstat
Disallow: *home
Disallow: *?p=1$
Disallow: /catalogsearch/
Disallow: /var/*
Disallow: *review/product*
Allow: *?p=
Sitemap: https://www.ruskniga.com/sitemap.xml

User-agent: BLEXBot
Disallow: /

User-agent: Baiduspider
User-agent: Baiduspider-video
User-agent: Baiduspider-image
Disallow: /

User-Agent: MJ12bot 
Disallow: / 

User-Agent: swebot 
Disallow: / 

User-agent: aihitbot 
Disallow: / 

User-agent: Exabot 
Disallow: / 

User-agent: sitebot 
Disallow: / 

User-agent: TwengaBot-Discover 
Disallow: / 

User-agent: Ezooms 
Disallow: / 

User-agent: Bender 
Disallow: / 

User-agent: discobot 
Disallow: / 

User-Agent: panscient.com 
Disallow: / 

User-Agent: searchwebengine.net 
Disallow: / 

User-agent: MLBot 
Disallow: / 

User-agent: NextGenSearchBot 
Disallow: / 

User-agent: Speedy 
Disallow: / 

User-agent: dotbot 
Disallow: / 

user-agent: AhrefsBot 
disallow: / 

User-agent: NerdByNature.Bot 
disallow: / 

User-agent: WBSearchBot 
Disallow: / 

User-agent: sistrix 
Disallow: / 

User-agent: SindiceBot 
Disallow: / 

User-agent: plukkie 
Disallow: / 

User-agent: findfiles.net 
Disallow: / 

User-agent: SeznamBot 
Disallow: / 

User-agent: Goodzer 
Disallow: / 

User-agent: BacklinkCrawler 
Disallow: / 

User-agent: lemurwebcrawler 
Disallow: / 

User-agent: gigabot 
Disallow: /

User-agent: discobot
Disallow: /

User-agent: FAST Enterprise Crawler 6
Disallow: /

#added by SD on 12/30/04 bot causing a lot of CF errors to occur
User-agent: Sensis.com.au Web Crawler
Disallow: / 

User-agent: worio bot heritrix
Disallow: /

User-agent: trovitBot
Disallow: /

## Do not crawl development files and folders: CVS, svn directories and dump files
Disallow: /CVS
Disallow: /*.svn$
Disallow: /*.idea$
Disallow: /*.sql$
Disallow: /*.tgz$
 
## GENERAL MAGENTO SETTINGS
 
## Do not crawl Magento admin page
Disallow: /admin/
 
## Do not crawl common Magento technical folders
Disallow: /app/
Disallow: /downloader/
Disallow: /errors/
Disallow: /includes/
Disallow: /lib/
Disallow: /pkginfo/
Disallow: /shell/

 
## Do not crawl common Magento files
Disallow: /api.php
Disallow: /cron.php
Disallow: /cron.sh
Disallow: /error_log
Disallow: /get.php
Disallow: /install.php
Disallow: /LICENSE.html
Disallow: /LICENSE.txt
Disallow: /LICENSE_AFL.txt
Disallow: /README.txt
Disallow: /RELEASE_NOTES.txt
 
## Do not crawl common server technical folders and files
Disallow: /cgi-bin/
Disallow: /cleanup.php
Disallow: /apc.php
Disallow: /memcache.php
Disallow: /phpinfo.php