Menu
▾
▴
[Phpbb-checkins] CVS: phpBB2/develop search_fill.php,1.6,1.7
|
From: Bart v. B. <ba...@us...> - 2002-01-03 11:06:25
|
Update of /cvsroot/phpbb/phpBB2/develop
In directory usw-pr-cvs1:/tmp/cvs-serv25202/develop
Modified Files:
search_fill.php
Log Message:
Moved all search functions to includes/search.php Paul; please review
Index: search_fill.php
===================================================================
RCS file: /cvsroot/phpbb/phpBB2/develop/search_fill.php,v
retrieving revision 1.6
retrieving revision 1.7
diff -C2 -r1.6 -r1.7
*** search_fill.php 2001/12/14 01:59:24 1.6
--- search_fill.php 2002/01/03 11:06:22 1.7
***************
*** 1,141 ****
<?php
- // Clean up an entry (posting), remove HTML, BBcode, stopwords, etc
- function clean_words($entry, &$search, &$replace)
- {
- // Weird, $init_match doesn't work with static when double quotes (") are used...
- static $init_match = array('^', '$', '&', '(', ')', '<', '>', '`', "'", '|', ',', '@', '_', '?', '%');
- static $init_replace = array(" ", " ", " ", " ", " ", " ", " ", " ", "", " ", " ", " ", " ", " ", " ");
-
- static $later_match = array("-", "~", "+", ".", "[", "]", "{", "}", ":", "\\", "/", "=", "#", "\"", ";", "*", "!");
- static $later_replace = array(" ", " ", " ", " ", " ", " ", " ", " ", " ", " " , " ", " ", " ", " ", " ", " ", " ");
-
- static $sgml_match = array(" ", "ß", "à", "á", "â", "ã", "ä", "å", "æ", "ç", "è", "é", "ê", "ë", "ì", "í", "î", "ï", "ð", "ñ", "ò", "ó", "ô", "õ", "ö", "ø", "ù", "ú", "û", "ü", "ý", "þ", "ÿ");
- static $sgml_replace = array(" ", "s", "a", "a", "a", "a", "a", "a", "a", "c", "e", "e", "e", "e", "i", "i", "i", "i", "o", "n", "o", "o", "o", "o", "o", "o", "u", "u", "u", "u", "y", "t", "y");
-
- static $accent_match = array("ß", "à", "á", "â", "ã", "ä", "å", "æ", "ç", "è", "é", "ê", "ë", "ì", "í", "î", "ï", "ð", "ñ", "ò", "ó", "ô", "õ", "ö", "ø", "ù", "ú", "û", "ü", "ý", "þ", "ÿ");
- static $accent_replace = array("s", "a", "a", "a", "a", "a", "a", "a", "c", "e", "e", "e", "e", "i", "i", "i", "i", "o", "n", "o", "o", "o", "o", "o", "o", "u", "u", "u", "u", "y", "t", "y");
-
- $entry = " " . strip_tags(strtolower($entry)) . " ";
-
- $entry = str_replace($sgml_match, $sgml_match, $entry);
- $entry = str_replace($accent_match, $accent_replace, $entry);
-
- // Replace line endings by a space
- $entry = preg_replace("/[\n\r]/is", " ", $entry);
- // Remove URL's
- $entry = preg_replace("/\b[a-z0-9]+:\/\/[a-z0-9\.\-]+(\/[a-z0-9\?\.%_\-\+=&\/]+)?/si", " ", $entry);
-
- // Filter out strange characters like ^, $, &, change "it's" to "its"
- // str_replace with arrays is buggy in some PHP versions so traverse the arrays manually ;(
- for($i = 0; $i < count($init_match); $i++)
- {
- $entry = str_replace($init_match[$i], $init_replace[$i], $entry);
- }
-
- // Quickly remove BBcode.
- $entry = preg_replace("/\[code:[0-9]+:[0-9a-z]{10,}\].*?\[\/code:[0-9]+:[0-9a-z]{10,}\]/is", " ", $entry);
- $entry = preg_replace("/\[img\].*?\[\/img\]/is", " ", $entry);
- $entry = preg_replace("/\[\/?[a-z\*=\+\-]+[0-9a-z]?(\:[a-z0-9]+)?:[a-z0-9]{10,}(\:[a-z0-9]+)?=?.*?\]/si", " ", $entry);
- // URLs
- $entry = preg_replace("/\[\/?[a-z\*]+[=\+\-]?[0-9a-z]+?:[a-z0-9]{10,}[=.*?]?\]/si", " ", $entry);
- $entry = preg_replace("/\[\/?url(=.*?)?\]/si", " ", $entry);
- // Numbers
- $entry = preg_replace("/\b[0-9]+\b/si", " ", $entry);
- // HTML entities like &1234;
- $entry = preg_replace("/\b&[a-z]+;\b/is", " ", $entry);
- // 'words' that consist of <2 or >50 characters are removed.
- $entry = preg_replace("/\b[a-z0-9]{1,2}?\b/si", " ", $entry);
- $entry = preg_replace("/\b[a-z0-9]{50,}?\b/si", " ", $entry);
-
- // Remove some more strange characters
- for($i = 0; $i < count($later_match); $i++)
- {
- $entry = str_replace($later_match[$i], $later_replace[$i], $entry);
- }
-
- // Remove stopwords
- $entry = preg_replace($search, $replace, $entry);
-
- return $entry;
- }
-
- function split_words(&$entry)
- {
- preg_match_all("/\b(\w[\w']*\w+|\w+?)\b/", $entry, $split_entries);
-
- return $split_entries[1];
- }
-
- function remove_common($percent, $delete_common = 0)
- {
- global $db;
-
- $sql = "SELECT COUNT(DISTINCT post_id) as total_posts
- FROM " . SEARCH_MATCH_TABLE;
- $result = $db->sql_query($sql);
- if( !$result )
- {
- $error = $db->sql_error();
- die("Couldn't get maximum post ID :: " . $sql . " :: " . $error['message']);
- }
- $total_posts = $db->sql_fetchrow($result);
- $total_posts = $total_posts['total_posts'];
-
- $common_threshold = floor($total_posts * ( $percent / 100 ));
-
- $sql = "SELECT word_id
- FROM " . SEARCH_MATCH_TABLE . "
- GROUP BY word_id
- HAVING count(word_id) > $common_threshold";
- $result = $db->sql_query($sql);
- if( !$result )
- {
- $error = $db->sql_error();
- die("Couldn't obtain common word list :: " . $sql . " :: " . $error['message']);
- }
- $common_words = $db->sql_numrows($result);
-
- while($row = $db->sql_fetchrow($result))
- {
- $common_word_ids[] = $row['word_id'];
- }
- $db->sql_freeresult($result);
-
- if(count($common_word_ids) != 0)
- {
- $common_word_ids = implode(',',$common_word_ids);
- }
- else
- {
- // We didn't remove any common words
- return 0;
- }
-
- $sql = "UPDATE ". SEARCH_WORD_TABLE ."
- SET word_common = 1
- WHERE word_id IN ($common_word_ids)";
- $result = $db->sql_query($sql);
- if( !$result )
- {
- $error = $db->sql_error();
- die("Couldn't delete word list entry :: " . $sql . " :: " . $error['message']);
- }
-
- if( $delete_common)
- {
- $sql = "DELETE FROM ".SEARCH_MATCH_TABLE."
- WHERE word_id IN ($common_word_ids)";
- $result = $db->sql_query($sql);
- if( !$result )
- {
- $error = $db->sql_error();
- die("Couldn't delete word match entry :: " . $sql . " :: " . $error['message']);
- }
- }
-
- return $common_words;
- }
-
set_time_limit(0);
$common_percent = 40; // Percentage of posts in which a word has to appear to be marked as common
--- 1,4 ----
***************
*** 148,151 ****
--- 11,15 ----
include($phpbb_root_path . 'includes/db.'.$phpEx);
include($phpbb_root_path . 'includes/functions.'.$phpEx);
+ include($phpbb_root_path . 'includes/search.'.$phpEx);
print "<html>\n<body>\n";
***************
*** 345,349 ****
print "<br>Removing common words (words that appear in more than $common_percent of the posts)<br>\n";
flush();
! print "Removed ". remove_common($common_percent, 1) ." words that where too common.<br>";
}
--- 209,213 ----
print "<br>Removing common words (words that appear in more than $common_percent of the posts)<br>\n";
flush();
! print "Removed ". remove_common_global($common_percent, 1) ." words that where too common.<br>";
}
|
