Dyomedea
/
owark
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Adding whois records

This commit is contained in:
Eric van der Vlist 2012-04-23 12:11:17 +02:00
parent 22c3028c38
commit 307b6d2a72
1 changed files with 3 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
archiver/pipelines/actions/crawler-beans-template.cxml
View File

@ -244,7 +244,7 @@ metadata.description=Basic crawl starting with useful defaults
<!-- <property name="digestContent" value="true" /> --> <!-- <property name="digestContent" value="true" /> -->
<!-- <property name="digestAlgorithm" value="sha1" /> --> <!-- <property name="digestAlgorithm" value="sha1" /> -->
</bean> </bean>
<!-- <bean id="fetchWhois" class="org.archive.modules.fetcher.FetchWhois"> <bean id="fetchWhois" class="org.archive.modules.fetcher.FetchWhois">
<property name="specialQueryTemplates"> <property name="specialQueryTemplates">
<map> <map>
<entry key="whois.verisign-grs.com" value="domain %s" /> <entry key="whois.verisign-grs.com" value="domain %s" />
@ -252,7 +252,7 @@ metadata.description=Basic crawl starting with useful defaults
<entry key="whois.denic.de" value="-T dn %s" /> <entry key="whois.denic.de" value="-T dn %s" />
</map> </map>
</property> </property>
</bean> --> </bean>
<bean id="fetchHttp" class="org.archive.modules.fetcher.FetchHTTP"> <bean id="fetchHttp" class="org.archive.modules.fetcher.FetchHTTP">
<!-- <property name="useHTTP11" value="false" /> --> <!-- <property name="useHTTP11" value="false" /> -->
<!-- <property name="maxLengthBytes" value="0" /> --> <!-- <property name="maxLengthBytes" value="0" /> -->
@ -313,7 +313,7 @@ metadata.description=Basic crawl starting with useful defaults
<ref bean="preconditions"/> <ref bean="preconditions"/>
<!-- ...fetch if DNS URI... --> <!-- ...fetch if DNS URI... -->
<ref bean="fetchDns"/> <ref bean="fetchDns"/>
<!-- <ref bean="fetchWhois"/> --> <ref bean="fetchWhois"/>
<!-- ...fetch if HTTP URI... --> <!-- ...fetch if HTTP URI... -->
<ref bean="fetchHttp"/> <ref bean="fetchHttp"/>
<!-- ...extract outlinks from HTTP headers... --> <!-- ...extract outlinks from HTTP headers... -->