nutch2.3.1 nutch-site.xml配置

<configuration> <property> <name>storage.data.store.class</name> <value>org.apache.gora.mongodb.store.MongoStore</value> </property> <property> <name>http.agent.name</name> <value>User-AgentMozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko</value> </property> <property> <name>plugin.includes</name> <value>protocol-httpclient|urlfilter-regex|index-(basic|more)|query-(basic|site|url|lang)|indexer-solr|nutch-extensionpoints|protocol-httpclient|urlfilter-regex|parse-(text|html|msexcel|msword|mspowerpoint|pdf)|summary-basic|scoring-opic|urlnormalizer-(pass|regex|basic)protocol-http|urlfilter-regex|parse-(html|tika|metatags)|index-(basic|anchor|more|metadata)</value> </property> <property> <name>plugin.folders</name> <value>plugins</value> </property> <property> <name>http.accept.language</name> <value>zh-CN,zh,ja-jp,en-us,en-gb,en;q=0.7,*;q=0.3</value> </property> <property> <name>parser.character.encoding.default</name> <value>utf-8</value> </property> <property> <name>http.content.limit</name> <value>-1</value> </property> <property> <name>file.content.limit</name> <value>-1</value> </property> <property> <name>parser.skip.truncated</name> <value>false</value> </property> <property> <name>anchorIndexingFilter.deduplicate</name> <value>true</value> </property> <property> <name>db.max.outlinks.per.page</name> <value>-1</value> </property> <property> <name>fetcher.server.delay</name> <value>0.9</value> </property> <property> <name>fetcher.server.min.delay</name> <value>0.8</value> </property> <property> <name>fetcher.threads.fetch</name> <value>25</value> </property <property> <name>fetcher.threads.per.queue</name> <value>3</value> </property> </configuration>