[Archive-access-cvs] archive-access/projects/nutch/conf ia-parse-plugins.xml,NONE,1.1.2.1 nutch-site.xml,1.24.2.4,1.24.2.5 parse-plugins.xml,1.1.2.1,NONE

[Doug C. <cu...@us...>]

Update of /cvsroot/archive-access/archive-access/projects/nutch/conf
In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv23098/conf

Modified Files:
      Tag: mapred
	nutch-site.xml 
Added Files:
      Tag: mapred
	ia-parse-plugins.xml 
Removed Files:
      Tag: mapred
	parse-plugins.xml 
Log Message:
Fix so that parse-pdf.sh is included in job jar, and is found relative to job jar.  Also fix so that parser plugin config overrides default.

--- parse-plugins.xml DELETED ---

--- NEW FILE: ia-parse-plugins.xml ---
<?xml version="1.0" encoding="UTF-8"?>
<!--
	Copyright 2005 The Apache Software Foundation
	
	Licensed under the Apache License, Version 2.0 (the "License");
	you may not use this file except in compliance with the License.
	You may obtain a copy of the License at
	
	http://www.apache.org/licenses/LICENSE-2.0
	
	Unless required by applicable law or agreed to in writing, software
	distributed under the License is distributed on an "AS IS" BASIS,
	WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	See the License for the specific language governing permissions and
	limitations under the License.
	
	Author     : mattmann 
	Description: This xml file represents a natural ordering for which parsing 
	plugin should get called for a particular mimeType. 
-->

<parse-plugins>

	<!--  by default if the mimeType is set to *, or 
		can't be determined, use parse-text -->
	<mimeType name="*">
		<plugin id="parse-text" />
	</mimeType>

	<mimeType name="application/java">
		<plugin id="parse-text" />
	</mimeType>

	<mimeType name="application/msword">
		<plugin id="parse-text" />
	</mimeType>

	<mimeType name="application/pdf">
		<plugin id="parse-ext" />
	</mimeType>

	<mimeType name="application/rss+xml">
		<plugin id="parse-text" />
	</mimeType>

	<mimeType name="application/vnd.ms-excel">
		<plugin id="parse-msexcel" />
	</mimeType>

	<mimeType name="application/vnd.ms-powerpoint">
		<plugin id="parse-mspowerpoint" />
	</mimeType>

	<mimeType name="application/vnd.wap.wbxml">
		<plugin id="parse-text" />
	</mimeType>

	<mimeType name="application/vnd.wap.wmlc">
		<plugin id="parse-text" />
	</mimeType>

	<mimeType name="application/vnd.wap.wmlscriptc">
		<plugin id="parse-text" />
	</mimeType>

	<mimeType name="application/xhtml+xml">
		<plugin id="parse-text" />
	</mimeType>

	<mimeType name="application/x-bzip2">
		<!--  try and parse it with the zip parser -->
		<plugin id="parse-zip" />
	</mimeType>

	<mimeType name="application/x-csh">
		<plugin id="parse-text" />
	</mimeType>

	<mimeType name="application/x-gzip">
		<!--  try and parse it with the zip parser -->
		<plugin id="parse-zip" />
	</mimeType>

	<mimeType name="application/x-javascript">
		<plugin id="parse-js" />
		<plugin id="parse-text" />
	</mimeType>

	<mimeType name="application/x-kword">
		<!--  try and parse it with the word parser -->
		<plugin id="parse-msword" />
	</mimeType>

	<mimeType name="application/x-kspread">
		<!--  try and parse it with the msexcel parser -->
		<plugin id="parse-msexcel" />
	</mimeType>

	<mimeType name="application/x-latex">
		<plugin id="parse-text" />
	</mimeType>

	<mimeType name="application/x-netcdf">
		<plugin id="parse-text" />
	</mimeType>

	<mimeType name="application/x-sh">
		<plugin id="parse-text" />
	</mimeType>

	<mimeType name="application/x-tcl">
		<plugin id="parse-text" />
	</mimeType>

	<mimeType name="application/x-tex">
		<plugin id="parse-text" />
	</mimeType>

	<mimeType name="application/x-texinfo">
		<plugin id="parse-text" />
	</mimeType>

	<mimeType name="application/x-troff">
		<plugin id="parse-text" />
	</mimeType>

	<mimeType name="application/x-troff-man">
		<plugin id="parse-text" />
	</mimeType>

	<mimeType name="application/x-troff-me">
		<plugin id="parse-text" />
	</mimeType>

	<mimeType name="application/x-troff-ms">
		<plugin id="parse-text" />
	</mimeType>

	<mimeType name="application/zip">
		<plugin id="parse-zip" />
	</mimeType>

	<mimeType name="message/news">
		<plugin id="parse-text" />
	</mimeType>

	<mimeType name="message/rfc822">
		<plugin id="parse-text" />
	</mimeType>

	<mimeType name="text/css">
		<plugin id="parse-text" />
	</mimeType>

	<mimeType name="text/html">
		<plugin id="parse-html" />
	</mimeType>

	<mimeType name="text/plain">
		<plugin id="parse-text" />
	</mimeType>

	<mimeType name="text/richtext">
		<plugin id="parse-rtf" />
		<plugin id="parse-msword" />
	</mimeType>

	<mimeType name="text/rtf">
		<plugin id="parse-rtf" />
		<plugin id="parse-msword" />
	</mimeType>

	<mimeType name="text/sgml">
		<plugin id="parse-html" />
		<plugin id="parse-text" />
	</mimeType>

	<mimeType name="text/tab-separated-values">
		<plugin id="parse-msexcel" />
		<plugin id="parse-text" />
	</mimeType>

	<mimeType name="text/vnd.wap.wml">
		<plugin id="parse-text" />
	</mimeType>

	<mimeType name="text/vnd.wap.wmlscript">
		<plugin id="parse-text" />
	</mimeType>

	<mimeType name="text/xml">
		<plugin id="parse-text" />
		<plugin id="parse-html" />
		<plugin id="parse-rss" />
	</mimeType>

	<mimeType name="text/x-setext">
		<plugin id="parse-text" />
	</mimeType>

       <!-- Types for parse-ext plugin: required for unit tests to pass. -->

	<mimeType name="application/vnd.nutch.example.cat">
		<plugin id="parse-ext" />
	</mimeType>

	<mimeType name="application/vnd.nutch.example.md5sum">
		<plugin id="parse-ext" />
	</mimeType>

</parse-plugins>

Index: nutch-site.xml
===================================================================
RCS file: /cvsroot/archive-access/archive-access/projects/nutch/conf/nutch-site.xml,v
retrieving revision 1.24.2.4
retrieving revision 1.24.2.5
diff -C2 -d -r1.24.2.4 -r1.24.2.5
*** nutch-site.xml	20 Oct 2005 23:30:48 -0000	1.24.2.4
--- nutch-site.xml	21 Oct 2005 20:53:57 -0000	1.24.2.5
***************
*** 7,21 ****
  <!-- NDFS -->
  
! <!-- <property> -->
! <!--   <name>fs.default.name</name> -->
! <!--   <value>ia109102.archive.org:8009</value> -->
! <!-- </property> -->
  
  <!-- MapReduce -->
  
! <!-- <property> -->
! <!--   <name>mapred.job.tracker</name> -->
! <!--   <value>ia109102.archive.org:8010</value> -->
! <!-- </property> -->
  
  <!-- Override a few Nutch defaults -->
--- 7,21 ----
  <!-- NDFS -->
  
! <property>
!   <name>fs.default.name</name>
!   <value>ia109102.archive.org:8009</value>
! </property>
  
  <!-- MapReduce -->
  
! <property>
!   <name>mapred.job.tracker</name>
!   <value>ia109102.archive.org:8010</value>
! </property>
  
  <!-- Override a few Nutch defaults -->
***************
*** 37,40 ****
--- 37,47 ----
  </property>
  
+ <property>
+   <name>parse.plugin.file</name>
+   <value>ia-parse-plugins.xml</value>
+   <description>The name of the file that defines the associations between
+   content-types and parsers.</description>
+ </property>
+ 
  <!-- don't truncate documents as much as by default -->
  <property>