# file: robots.txt,v 1.0 2002/09/23 created by hellojan # www.ccfgroup.com # 按照robots.txt的标准写法,规定一些不允许爬虫爬的页面或目录。 # robots.txt 的写法参照 # Format is: # User-agent: # Disallow: | # ----------------------------------------------------------------------------- User-agent: * #Disallow: /member/ Disallow: /graph/ Disallow: /includes/ #Disallow: /newscenter/news_search.php #Disallow: /newscenter/news.inc.php #Disallow: /newscenter/newsview.php #Disallow: /newscenter/search.php #Disallow: /newscenter/showindex.php Disallow: /newscenter/pbview.php Disallow: /informs/index_prod.php Disallow: /informs/newsintfixfile.php Disallow: /informs/showindex.php Disallow: /informs/showmrkinfo.php Disallow: /enterprise/add.php Disallow: /enterprise/enterprise_add.php Disallow: /enterprise/enterprise_list.php Disallow: /enterprise/list_more.php