mirror of
https://tt-rss.org/git/tt-rss.git
synced 2024-07-05 13:20:55 +02:00
298 lines
8.9 KiB
PHP
298 lines
8.9 KiB
PHP
<?php
|
|
/*
|
|
***** BEGIN LICENSE BLOCK *****
|
|
This file is part of PHP Naive Bayesian Filter.
|
|
|
|
The Initial Developer of the Original Code is
|
|
Loic d'Anterroches [loic_at_xhtml.net].
|
|
Portions created by the Initial Developer are Copyright (C) 2003
|
|
the Initial Developer. All Rights Reserved.
|
|
|
|
Contributor(s):
|
|
See the source
|
|
|
|
PHP Naive Bayesian Filter is free software; you can redistribute it
|
|
and/or modify it under the terms of the GNU General Public License as
|
|
published by the Free Software Foundation; either version 2 of
|
|
the License, or (at your option) any later version.
|
|
|
|
PHP Naive Bayesian Filter is distributed in the hope that it will
|
|
be useful, but WITHOUT ANY WARRANTY; without even the implied
|
|
warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
|
|
See the GNU General Public License for more details.
|
|
|
|
You should have received a copy of the GNU General Public License
|
|
along with Foobar; if not, write to the Free Software
|
|
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
|
|
|
Alternatively, the contents of this file may be used under the terms of
|
|
the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
|
|
in which case the provisions of the LGPL are applicable instead
|
|
of those above.
|
|
|
|
***** END LICENSE BLOCK *****
|
|
*/
|
|
|
|
class NaiveBayesian {
|
|
/** min token length for it to be taken into consideration */
|
|
var $min_token_length = 3;
|
|
/** max token length for it to be taken into consideration */
|
|
var $max_token_length = 15;
|
|
/** list of token to ignore
|
|
@see getIgnoreList()
|
|
*/
|
|
var $ignore_list = array();
|
|
/** storage object
|
|
@see class NaiveBayesianStorage
|
|
*/
|
|
var $nbs = null;
|
|
|
|
function NaiveBayesian($nbs) {
|
|
$this->nbs = $nbs;
|
|
|
|
return true;
|
|
}
|
|
|
|
/** categorize a document.
|
|
Get list of categories in which the document can be categorized
|
|
with a score for each category.
|
|
|
|
@return array keys = category ids, values = scores
|
|
@param string document
|
|
*/
|
|
function categorize($document) {
|
|
$scores = array();
|
|
$categories = $this->nbs->getCategories();
|
|
$tokens = $this->_getTokens($document);
|
|
|
|
// calculate the score in each category
|
|
$total_words = 0;
|
|
$ncat = 0;
|
|
|
|
while (list($category, $data) = each($categories)) {
|
|
$total_words += $data['word_count'];
|
|
$ncat++;
|
|
}
|
|
|
|
reset($categories);
|
|
|
|
while (list($category, $data) = each($categories)) {
|
|
$scores[$category] = $data['probability'];
|
|
// small probability for a word not in the category
|
|
// maybe putting 1.0 as a 'no effect' word can also be good
|
|
|
|
if ($data['word_count'] > 0)
|
|
$small_proba = 1.0 / ($data['word_count'] * 2);
|
|
else
|
|
$small_proba = 0;
|
|
|
|
reset($tokens);
|
|
|
|
while (list($token, $count) = each($tokens)) {
|
|
|
|
if ($this->nbs->wordExists($token)) {
|
|
$word = $this->nbs->getWord($token, $category);
|
|
|
|
if ($word['count']) {
|
|
$proba = $word['count'] / $data['word_count'];
|
|
}
|
|
else {
|
|
$proba = $small_proba;
|
|
}
|
|
|
|
$scores[$category] *= pow($proba, $count) * pow($total_words / $ncat, $count);
|
|
// pow($total_words/$ncat, $count) is here to avoid underflow.
|
|
|
|
}
|
|
}
|
|
}
|
|
|
|
return $this->_rescale($scores);
|
|
}
|
|
|
|
/** training against a document.
|
|
Set a document as being in a specific category. The document becomes a reference
|
|
and is saved in the table of references. After a set of training is done
|
|
the updateProbabilities() function must be run.
|
|
|
|
@see updateProbabilities()
|
|
@see untrain()
|
|
@return bool success
|
|
@param string document id, must be unique
|
|
@param string category_id the category id in which the document should be
|
|
@param string content of the document
|
|
*/
|
|
function train($doc_id, $category_id, $content) {
|
|
$ret = false;
|
|
|
|
|
|
// if this doc_id already trained, no trained
|
|
if (!$this->nbs->getReference($doc_id, false)) {
|
|
|
|
$tokens = $this->_getTokens($content);
|
|
|
|
while (list($token, $count) = each($tokens)) {
|
|
$this->nbs->updateWord($token, $count, $category_id);
|
|
}
|
|
|
|
$this->nbs->saveReference($doc_id, $category_id, $content);
|
|
|
|
$ret = true;
|
|
}
|
|
else {
|
|
$ret = false;
|
|
}
|
|
|
|
return $ret;
|
|
}
|
|
|
|
/** untraining of a document.
|
|
To remove just one document from the references.
|
|
|
|
@see updateProbabilities()
|
|
@see untrain()
|
|
@return bool success
|
|
@param string document id, must be unique
|
|
*/
|
|
function untrain($doc_id) {
|
|
$ref = $this->nbs->getReference($doc_id);
|
|
|
|
if (isset($ref['content'])) {
|
|
|
|
$tokens = $this->_getTokens($ref['content']);
|
|
|
|
while (list($token, $count) = each($tokens)) {
|
|
$this->nbs->removeWord($token, $count, $ref['category_id']);
|
|
}
|
|
|
|
$this->nbs->removeReference($doc_id);
|
|
|
|
return true;
|
|
} else {
|
|
return false;
|
|
}
|
|
}
|
|
|
|
/** rescale the results between 0 and 1.
|
|
|
|
@author Ken Williams, ken@mathforum.org
|
|
@see categorize()
|
|
@return array normalized scores (keys => category, values => scores)
|
|
@param array scores (keys => category, values => scores)
|
|
*/
|
|
function _rescale($scores) {
|
|
// Scale everything back to a reasonable area in
|
|
// logspace (near zero), un-loggify, and normalize
|
|
$total = 0.0;
|
|
$max = 0.0;
|
|
reset($scores);
|
|
|
|
while (list($cat, $score) = each($scores)) {
|
|
if ($score >= $max)
|
|
$max = $score;
|
|
}
|
|
|
|
reset($scores);
|
|
while (list($cat, $score) = each($scores)) {
|
|
$scores[$cat] = (float) exp($score - $max);
|
|
$total += (float) pow($scores[$cat], 2);
|
|
}
|
|
|
|
$total = (float) sqrt($total);
|
|
|
|
reset($scores);
|
|
while (list($cat, $score) = each($scores)) {
|
|
$scores[$cat] = (float) $scores[$cat] / $total;
|
|
}
|
|
reset($scores);
|
|
|
|
return $scores;
|
|
}
|
|
|
|
/** update the probabilities of the categories and word count.
|
|
This function must be run after a set of training
|
|
|
|
@see train()
|
|
@see untrain()
|
|
@return bool sucess
|
|
*/
|
|
function updateProbabilities() {
|
|
// this function is really only database manipulation
|
|
// that is why all is done in the NaiveBayesianStorage
|
|
return $this->nbs->updateProbabilities();
|
|
}
|
|
|
|
/** Get the list of token to ignore.
|
|
@return array ignore list
|
|
*/
|
|
function getIgnoreList() {
|
|
//return array('the', 'that', 'you', 'for', 'and');
|
|
|
|
// https://en.wikipedia.org/wiki/Most_common_words_in_English
|
|
return array('the', 'be', 'to', 'of', 'and', 'a', 'in', 'that', 'have', 'I', 'it', 'for', 'not', 'on', 'with',
|
|
'he', 'as', 'you', 'do', 'at', 'this', 'but', 'his', 'by', 'from', 'they', 'we', 'say', 'her',
|
|
'she', 'or', 'an', 'will', 'my', 'one', 'all', 'would', 'there', 'their', 'what', 'so', 'up',
|
|
'out', 'if', 'about', 'who', 'get', 'which', 'go', 'me', 'when', 'make', 'can', 'like', 'time',
|
|
'no', 'just', 'him', 'know', 'take', 'people', 'into', 'year', 'your', 'good', 'some', 'could',
|
|
'them', 'see', 'other', 'than', 'then', 'now', 'look', 'only', 'come', 'its', 'over', 'think',
|
|
'also', 'back', 'after', 'use', 'two', 'how', 'our', 'work', 'first', 'well', 'way', 'even',
|
|
'new', 'want', 'because', 'any', 'these', 'give', 'day', 'most', 'us', 'read', 'more');
|
|
|
|
}
|
|
|
|
/** get the tokens from a string
|
|
|
|
@author James Seng. [http://james.seng.cc/] (based on his perl version)
|
|
|
|
@return array tokens
|
|
@param string the string to get the tokens from
|
|
*/
|
|
function _getTokens($string) {
|
|
$rawtokens = array();
|
|
$tokens = array();
|
|
//$string = $this->_cleanString($string);
|
|
|
|
if (count(0 >= $this->ignore_list)) {
|
|
$this->ignore_list = $this->getIgnoreList();
|
|
}
|
|
|
|
$rawtokens = preg_split("/[\(\),:\.;\t\r\n ]/", $string, -1, PREG_SPLIT_NO_EMPTY);
|
|
|
|
// remove some tokens
|
|
while (list(, $token) = each($rawtokens)) {
|
|
$token = trim($token);
|
|
if (!(('' == $token) || (mb_strpos($token, "&") !== FALSE) || (mb_strlen($token) < $this->min_token_length) || (mb_strlen($token) > $this->max_token_length) || (preg_match('/^[0-9]+$/', $token)) || (in_array($token, $this->ignore_list)))) {
|
|
$tokens[$token]++;
|
|
}
|
|
}
|
|
|
|
return $tokens;
|
|
}
|
|
|
|
/** clean a string from the diacritics
|
|
|
|
@author Antoine Bajolet [phpdig_at_toiletoine.net]
|
|
@author SPIP [http://uzine.net/spip/]
|
|
|
|
@return string clean string
|
|
@param string string with accents
|
|
*/
|
|
function _cleanString($string) {
|
|
$diac = /* A */ chr(192) . chr(193) . chr(194) . chr(195) . chr(196) . chr(197) .
|
|
/* a */ chr(224) . chr(225) . chr(226) . chr(227) . chr(228) . chr(229) .
|
|
/* O */ chr(210) . chr(211) . chr(212) . chr(213) . chr(214) . chr(216) .
|
|
/* o */ chr(242) . chr(243) . chr(244) . chr(245) . chr(246) . chr(248) .
|
|
/* E */ chr(200) . chr(201) . chr(202) . chr(203) .
|
|
/* e */ chr(232) . chr(233) . chr(234) . chr(235) .
|
|
/* Cc */ chr(199) . chr(231) .
|
|
/* I */ chr(204) . chr(205) . chr(206) . chr(207) .
|
|
/* i */ chr(236) . chr(237) . chr(238) . chr(239) .
|
|
/* U */ chr(217) . chr(218) . chr(219) . chr(220) .
|
|
/* u */ chr(249) . chr(250) . chr(251) . chr(252) .
|
|
/* yNn */ chr(255) . chr(209) . chr(241);
|
|
|
|
return strtolower(strtr($string, $diac, 'AAAAAAaaaaaaOOOOOOooooooEEEEeeeeCcIIIIiiiiUUUUuuuuyNn'));
|
|
}
|
|
|
|
}
|