user-agent: wsscrawler disallow: user-agent: agosearch disallow: User-agent: * Disallow: /