[frontierkernel-commits] odbs/mainResponderRoot/mainResponder/search/server indexOnePage,1.1.1.1,1.2

[creedon <icr...@us...>]

Update of /cvsroot/frontierkernel/odbs/mainResponderRoot/mainResponder/search/server
In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv15628

Modified Files:
	indexOnePage 
Log Message:
replace searchEngine.cleanText with mainResponder.search.utilities.cleanText

Index: indexOnePage
===================================================================
RCS file: /cvsroot/frontierkernel/odbs/mainResponderRoot/mainResponder/search/server/indexOnePage,v
retrieving revision 1.1.1.1
retrieving revision 1.2
diff -C2 -d -r1.1.1.1 -r1.2
*** indexOnePage	26 Mar 2005 21:46:12 -0000	1.1.1.1
--- indexOnePage	13 Aug 2005 20:07:57 -0000	1.2
***************
*** 1,3 ****
  FrontierVcsFile:1:scpt:mainResponder.search.server.indexOnePage
  
! on indexOnePage (url, title, pageText, adrStopWords=nil, indexName=nil) {
	ÇIndex one page.
		ÇChanges:
			Ç8/20/99; 11:41:41 AM by PBS
				ÇIndex all words that are not in the stop words table. Don't ignore numbers. Don't ignore words that are shorter than 3 characters.
			Ç07/18/00; 10:46:01 PM by PBS
				ÇInstead of using string.nthField, just keep deleting from the leading edge of the text, and index the first word. Performance optimization.
	
	local (lowerTitle = string.lower (title));
	local (pageName);
	local (lowerAdr = string.lower (url));
	local (lowerParent);
	local (i, j, numFields);
	local (flMessages = true);
	local (indexPath = mainResponder.search.utilities.getIndexPath (indexName));
	local (adrIndex = @[indexPath].index);
	
	if adrStopWords == nil {
		adrStopWords = @searchEngine.data.stopWords};
	
	on doMessage (s) {
		if flMessages {
			msg (s)}};
	
	on getLastField (s) {
		local (ctfields = string.countFields (s, '/'));
		s = string.nthField (s, '/', ctfields);
		return (s)};
	
	bundle { //get the page name
		pageName = url;
		pageName = getLastField (pageName)};
	
	bundle { //get the name of the parent folder
		lowerParent = string.lower (url);
		lowerParent = string.popSuffix (lowerParent, '/');
		lowerParent = getLastField (lowerParent)};
	
	bundle { //clean the text before indexing
		pageText = searchEngine.cleanText (pageText)};
	
	ÇnumFields = string.countFields (pageText,  ' ') //count potential words
	local (ct = 0);
	Çfor i = 1 to numFields //loop through every potential word
		Çlocal (oneWord = string.nthField (pageText, ' ', i))
		Ç
		Çif oneWord beginsWith '#'
			Çcontinue //it's a directive
		Ç
		ÇoneWord = string.dropNonAlphas (oneWord)
		ÇoneWord = string.lower (oneWord)
		ÇoneWord = string.popTrailing (oneWord, 's') //pop off trailing s's
		ÇoneWord = string.trimWhiteSpace (oneWord)
		Çif oneWord == ""
			Çcontinue
		Ç
		ÇPBS 8/20/99: commented this out -- we now index all words that are not in the stop words table.
		Çif sizeOf (oneWord) < 3 //we don't index very short words, except for "op" and "wp"
			Çif oneWord != "op" and oneWord != "wp" and oneWord != "ii"
				Çcontinue
		Ç
		Çif not (searchEngine.checkStopWords (oneWord, adrStopWords)) //check if this is on the list of words not to index
			Çcontinue //don't index
		Ç
		Çsys.systemTask ()
		ÇdoMessage (oneWord)
		Ç
		Çlocal (firstLetter = string.mid (oneWord, 1, 1))
		Ç
		ÇPBS 8/20/99: a word can start with a number. At this point we have letters and numbers only, so this check can be eliminated.
		Çif char (firstLetter) < 'a' or char (firstLetter) > 'z'
			Çcontinue
		Ç
		Çlocal (adrLetter = @adrIndex^.[firstLetter]) //address of this letter's table in the index
		Çif not defined (adrLetter^)
			Çnew (tableType, adrLetter)
		Ç
		Çlocal (adrWord = @adrLetter^.[oneWord]) //address of this word in the index
		Çif not defined (adrWord^)
			Çnew (tableType, adrWord)
		Ç
		Çlocal (adrPageCount = @adrWord^.[url]) //address of the count for this page in this word
		Çif defined (adrPageCount^)
			ÇadrPageCount^ = adrPageCount^ + 1 //this is a frequency count (plus the relevancy ranking, see below)
		Çelse
			ÇadrPageCount^ = 1 //first occurence of this word in the page
		Ç
		Çbundle //do relevancy ranking
			Çif lowerAdr contains oneWord //if the address of the page contains the word, add 100
				Çif adrPageCount^ < 100
					ÇadrPageCount^ = adrPageCount^ + 100
			Çif string.lower (pageName) contains oneWord //if the name of the page contains the word, add 500
				Çif adrPageCount^ < 500
					ÇadrPageCount^ = adrPageCount^ + 500
			Çif lowerParent == oneWord //if the name of the parent table equals the word and this is the default page, add 1000
				Çif pageName contains "default" or pageName contains "index"
					Çif adrPageCount^ < 1000
						ÇadrPageCount^ = adrPageCount^ + 1000
			Çif lowerTitle contains oneWord //if the title of the page contains the word, add 2000
				Çif adrPageCount^ < 2000
					ÇadrPageCount^ = adrPageCount^ + 2000
		Ç
		Çsys.systemTask ()
		ÇdoMessage ("Indexing: " + url + ": " + oneWord)
	while sizeOf (pageText) > 0 { //PBS 07/18/00: loop through every potential word
		pageText = string.trimWhiteSpace (pageText);
		local (oneWord = string.nthField (pageText, ' ', 1));
		local (wordSize = sizeOf (oneWord));
		
		if oneWord beginsWith '#' {
			pageText = string.delete (pageText, 1, wordSize);
			continue}; //it's a directive
		
		oneWord = string.dropNonAlphas (oneWord);
		oneWord = string.lower (oneWord);
		oneWord = string.popTrailing (oneWord, 's'); //pop off trailing s's
		oneWord = string.trimWhiteSpace (oneWord);
		if oneWord == "" {
			pageText = string.delete (pageText, 1, wordSize);
			continue};
		
		if not (searchEngine.checkStopWords (oneWord, adrStopWords)) { //check if this is on the list of words not to index
			pageText = string.delete (pageText, 1, wordSize);
			continue}; //don't index
		
		local (firstLetter = string.mid (oneWord, 1, 1));
		
		local (adrLetter = @adrIndex^.[firstLetter]); //address of this letter's table in the index
		if not defined (adrLetter^) {
			new (tableType, adrLetter)};
		
		local (adrWord = @adrLetter^.[oneWord]); //address of this word in the index
		if not defined (adrWord^) {
			new (tableType, adrWord)};
		
		local (adrPageCount = @adrWord^.[url]); //address of the count for this page in this word
		if defined (adrPageCount^) {
			adrPageCount^ = adrPageCount^ + 1} //this is a frequency count (plus the relevancy ranking, see below)
		else {
			adrPageCount^ = 1}; //first occurence of this word in the page
		
		bundle { //do relevancy ranking
			if lowerAdr contains oneWord { //if the address of the page contains the word, add 100
				if adrPageCount^ < 100 {
					adrPageCount^ = adrPageCount^ + 100}};
			if string.lower (pageName) contains oneWord { //if the name of the page contains the word, add 500
				if adrPageCount^ < 500 {
					adrPageCount^ = adrPageCount^ + 500}};
			if lowerParent == oneWord { //if the name of the parent table equals the word and this is the default page, add 1000
				if pageName contains "default" or pageName contains "index" {
					if adrPageCount^ < 1000 {
						adrPageCount^ = adrPageCount^ + 1000}}};
			if lowerTitle contains oneWord { //if the title of the page contains the word, add 2000
				if adrPageCount^ < 2000 {
					adrPageCount^ = adrPageCount^ + 2000}}};
		
		pageText = string.delete (pageText, 1, wordSize);
		
		Çct++
		Çif ct > 500 //relax on occassion
			Çthread.sleepFor (0)
			Çct = 0};
	
	msg ("");
	
	return (true)}
\ No newline at end of file
--- 1,3 ----
  FrontierVcsFile:1:scpt:mainResponder.search.server.indexOnePage
  
! on indexOnePage (url, title, pageText, adrStopWords=nil, indexName=nil) {
	ÇIndex one page.
	ÇChanges
		Ç8/12/05; 11:14:07 AM by TAC
			Çreplace searchEngine.cleanText with mainResponder.search.utilities.cleanText
		Ç07/18/00; 10:46:01 PM by PBS
			ÇInstead of using string.nthField, just keep deleting from the leading edge of the text, and index the first word. Performance optimization.
		Ç8/20/99; 11:41:41 AM by PBS
			ÇIndex all words that are not in the stop words table. Don't ignore numbers. Don't ignore words that are shorter than 3 characters.
	
	local (lowerTitle = string.lower (title));
	local (pageName);
	local (lowerAdr = string.lower (url));
	local (lowerParent);
	local (i, j, numFields);
	local (flMessages = true);
	local (indexPath = mainResponder.search.utilities.getIndexPath (indexName));
	local (adrIndex = @[indexPath].index);
	
	if adrStopWords == nil {
		adrStopWords = @searchEngine.data.stopWords};
	
	on doMessage (s) {
		if flMessages {
			msg (s)}};
	
	on getLastField (s) {
		local (ctfields = string.countFields (s, '/'));
		s = string.nthField (s, '/', ctfields);
		return (s)};
	
	bundle { //get the page name
		pageName = url;
		pageName = getLastField (pageName)};
	
	bundle { //get the name of the parent folder
		lowerParent = string.lower (url);
		lowerParent = string.popSuffix (lowerParent, '/');
		lowerParent = getLastField (lowerParent)};
	
	bundle { //clean the text before indexing
		pageText = mainResponder.search.utilities.cleanText (pageText)};
	
	ÇnumFields = string.countFields (pageText,  ' ') //count potential words
	local (ct = 0);
	Çfor i = 1 to numFields //loop through every potential word
		Çlocal (oneWord = string.nthField (pageText, ' ', i))
		Ç
		Çif oneWord beginsWith '#'
			Çcontinue //it's a directive
		Ç
		ÇoneWord = string.dropNonAlphas (oneWord)
		ÇoneWord = string.lower (oneWord)
		ÇoneWord = string.popTrailing (oneWord, 's') //pop off trailing s's
		ÇoneWord = string.trimWhiteSpace (oneWord)
		Çif oneWord == ""
			Çcontinue
		Ç
		ÇPBS 8/20/99: commented this out -- we now index all words that are not in the stop words table.
		Çif sizeOf (oneWord) < 3 //we don't index very short words, except for "op" and "wp"
			Çif oneWord != "op" and oneWord != "wp" and oneWord != "ii"
				Çcontinue
		Ç
		Çif not (searchEngine.checkStopWords (oneWord, adrStopWords)) //check if this is on the list of words not to index
			Çcontinue //don't index
		Ç
		Çsys.systemTask ()
		ÇdoMessage (oneWord)
		Ç
		Çlocal (firstLetter = string.mid (oneWord, 1, 1))
		Ç
		ÇPBS 8/20/99: a word can start with a number. At this point we have letters and numbers only, so this check can be eliminated.
		Çif char (firstLetter) < 'a' or char (firstLetter) > 'z'
			Çcontinue
		Ç
		Çlocal (adrLetter = @adrIndex^.[firstLetter]) //address of this letter's table in the index
		Çif not defined (adrLetter^)
			Çnew (tableType, adrLetter)
		Ç
		Çlocal (adrWord = @adrLetter^.[oneWord]) //address of this word in the index
		Çif not defined (adrWord^)
			Çnew (tableType, adrWord)
		Ç
		Çlocal (adrPageCount = @adrWord^.[url]) //address of the count for this page in this word
		Çif defined (adrPageCount^)
			ÇadrPageCount^ = adrPageCount^ + 1 //this is a frequency count (plus the relevancy ranking, see below)
		Çelse
			ÇadrPageCount^ = 1 //first occurence of this word in the page
		Ç
		Çbundle //do relevancy ranking
			Çif lowerAdr contains oneWord //if the address of the page contains the word, add 100
				Çif adrPageCount^ < 100
					ÇadrPageCount^ = adrPageCount^ + 100
			Çif string.lower (pageName) contains oneWord //if the name of the page contains the word, add 500
				Çif adrPageCount^ < 500
					ÇadrPageCount^ = adrPageCount^ + 500
			Çif lowerParent == oneWord //if the name of the parent table equals the word and this is the default page, add 1000
				Çif pageName contains "default" or pageName contains "index"
					Çif adrPageCount^ < 1000
						ÇadrPageCount^ = adrPageCount^ + 1000
			Çif lowerTitle contains oneWord //if the title of the page contains the word, add 2000
				Çif adrPageCount^ < 2000
					ÇadrPageCount^ = adrPageCount^ + 2000
		Ç
		Çsys.systemTask ()
		ÇdoMessage ("Indexing: " + url + ": " + oneWord)
	while sizeOf (pageText) > 0 { //PBS 07/18/00: loop through every potential word
		pageText = string.trimWhiteSpace (pageText);
		local (oneWord = string.nthField (pageText, ' ', 1));
		local (wordSize = sizeOf (oneWord));
		
		if oneWord beginsWith '#' {
			pageText = string.delete (pageText, 1, wordSize);
			continue}; //it's a directive
		
		oneWord = string.dropNonAlphas (oneWord);
		oneWord = string.lower (oneWord);
		oneWord = string.popTrailing (oneWord, 's'); //pop off trailing s's
		oneWord = string.trimWhiteSpace (oneWord);
		if oneWord == "" {
			pageText = string.delete (pageText, 1, wordSize);
			continue};
		
		if not (searchEngine.checkStopWords (oneWord, adrStopWords)) { //check if this is on the list of words not to index
			pageText = string.delete (pageText, 1, wordSize);
			continue}; //don't index
		
		local (firstLetter = string.mid (oneWord, 1, 1));
		
		local (adrLetter = @adrIndex^.[firstLetter]); //address of this letter's table in the index
		if not defined (adrLetter^) {
			new (tableType, adrLetter)};
		
		local (adrWord = @adrLetter^.[oneWord]); //address of this word in the index
		if not defined (adrWord^) {
			new (tableType, adrWord)};
		
		local (adrPageCount = @adrWord^.[url]); //address of the count for this page in this word
		if defined (adrPageCount^) {
			adrPageCount^ = adrPageCount^ + 1} //this is a frequency count (plus the relevancy ranking, see below)
		else {
			adrPageCount^ = 1}; //first occurence of this word in the page
		
		bundle { //do relevancy ranking
			if lowerAdr contains oneWord { //if the address of the page contains the word, add 100
				if adrPageCount^ < 100 {
					adrPageCount^ = adrPageCount^ + 100}};
			if string.lower (pageName) contains oneWord { //if the name of the page contains the word, add 500
				if adrPageCount^ < 500 {
					adrPageCount^ = adrPageCount^ + 500}};
			if lowerParent == oneWord { //if the name of the parent table equals the word and this is the default page, add 1000
				if pageName contains "default" or pageName contains "index" {
					if adrPageCount^ < 1000 {
						adrPageCount^ = adrPageCount^ + 1000}}};
			if lowerTitle contains oneWord { //if the title of the page contains the word, add 2000
				if adrPageCount^ < 2000 {
					adrPageCount^ = adrPageCount^ + 2000}}};
		
		pageText = string.delete (pageText, 1, wordSize);
		
		Çct++
		Çif ct > 500 //relax on occassion
			Çthread.sleepFor (0)
			Çct = 0};};
	
	msg ("");
	
	return (true)}
\ No newline at end of file