valvin
/
ttrss
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feedparser: remove horrible hacks for feeds with broken encoding

This commit is contained in:
Andrew Dolgov 2019-04-01 09:02:05 +03:00
parent cb0c81729d
commit 1a484ec3f5
1 changed files with 0 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

54
classes/feedparser.php
View File

@ -13,20 +13,6 @@ class FeedParser {
const FEED_RSS = 1; const FEED_RSS = 1;
const FEED_ATOM = 2; const FEED_ATOM = 2;
function normalize_encoding($data) {
if (preg_match('/^(<\?xml[\t\n\r ].*?encoding[\t\n\r ]*=[\t\n\r ]*["\'])(.+?)(["\'].*?\?>)/s', $data, $matches) === 1) {
$encoding = strtolower($matches[2]);
if (in_array($encoding, array_map('strtolower', mb_list_encodings())))
$data = mb_convert_encoding($data, 'UTF-8', $encoding);
$data = preg_replace('/^<\?xml[\t\n\r ].*?\?>/s', $matches[1] . "UTF-8" . $matches[3] , $data);
}
return $data;
}
function __construct($data) { function __construct($data) {
libxml_use_internal_errors(true); libxml_use_internal_errors(true);
libxml_clear_errors(); libxml_clear_errors();
@ -37,46 +23,6 @@ class FeedParser {
$error = libxml_get_last_error(); $error = libxml_get_last_error();
// libxml compiled without iconv?
if ($error && $error->code == 32) {
$data = $this->normalize_encoding($data);
if ($data) {
libxml_clear_errors();
$this->doc = new DOMDocument();
$this->doc->loadXML($data);
$error = libxml_get_last_error();
}
}
// some terrible invalid unicode entity?
if ($error) {
foreach (libxml_get_errors() as $err) {
if ($err->code == 9) {
// if the source feed is not in utf8, next conversion will fail
$data = $this->normalize_encoding($data);
// remove dangling bytes
$data = mb_convert_encoding($data, 'UTF-8', 'UTF-8');
// apparently not all UTF-8 characters are valid for XML
$data = preg_replace('/[^\x{0009}\x{000a}\x{000d}\x{0020}-\x{D7FF}\x{E000}-\x{FFFD}]+/u', ' ', $data);
if ($data) {
libxml_clear_errors();
$this->doc = new DOMDocument();
$this->doc->loadXML($data);
$error = libxml_get_last_error();
}
break;
}
}
}
if ($error) { if ($error) {
foreach (libxml_get_errors() as $error) { foreach (libxml_get_errors() as $error) {
if ($error->level == LIBXML_ERR_FATAL) { if ($error->level == LIBXML_ERR_FATAL) {