valvin
/
ttrss
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
ttrss/classes/diskcache.php

417 lines
22 KiB
PHP
Raw Normal View History

add classes/diskcache
2019-08-13 09:04:36 +00:00
<?php
expire_cached_files to DiskCache::expire()
2019-08-13 11:13:42 +00:00
class DiskCache {
add classes/diskcache
2019-08-13 09:04:36 +00:00
private $dir;
DiskCache: append fake file extension when sending cached files based on mime type to make saving files easier
2020-05-12 10:28:54 +00:00
// https://stackoverflow.com/a/53662733
private $mimeMap = [
'video/3gpp2' => '3g2',
'video/3gp' => '3gp',
'video/3gpp' => '3gp',
'application/x-compressed' => '7zip',
'audio/x-acc' => 'aac',
'audio/ac3' => 'ac3',
'application/postscript' => 'ai',
'audio/x-aiff' => 'aif',
'audio/aiff' => 'aif',
'audio/x-au' => 'au',
'video/x-msvideo' => 'avi',
'video/msvideo' => 'avi',
'video/avi' => 'avi',
'application/x-troff-msvideo' => 'avi',
'application/macbinary' => 'bin',
'application/mac-binary' => 'bin',
'application/x-binary' => 'bin',
'application/x-macbinary' => 'bin',
'image/bmp' => 'bmp',
'image/x-bmp' => 'bmp',
'image/x-bitmap' => 'bmp',
'image/x-xbitmap' => 'bmp',
'image/x-win-bitmap' => 'bmp',
'image/x-windows-bmp' => 'bmp',
'image/ms-bmp' => 'bmp',
'image/x-ms-bmp' => 'bmp',
'application/bmp' => 'bmp',
'application/x-bmp' => 'bmp',
'application/x-win-bitmap' => 'bmp',
'application/cdr' => 'cdr',
'application/coreldraw' => 'cdr',
'application/x-cdr' => 'cdr',
'application/x-coreldraw' => 'cdr',
'image/cdr' => 'cdr',
'image/x-cdr' => 'cdr',
'zz-application/zz-winassoc-cdr' => 'cdr',
'application/mac-compactpro' => 'cpt',
'application/pkix-crl' => 'crl',
'application/pkcs-crl' => 'crl',
'application/x-x509-ca-cert' => 'crt',
'application/pkix-cert' => 'crt',
'text/css' => 'css',
'text/x-comma-separated-values' => 'csv',
'text/comma-separated-values' => 'csv',
'application/vnd.msexcel' => 'csv',
'application/x-director' => 'dcr',
'application/vnd.openxmlformats-officedocument.wordprocessingml.document' => 'docx',
'application/x-dvi' => 'dvi',
'message/rfc822' => 'eml',
'application/x-msdownload' => 'exe',
'video/x-f4v' => 'f4v',
'audio/x-flac' => 'flac',
'video/x-flv' => 'flv',
'image/gif' => 'gif',
'application/gpg-keys' => 'gpg',
'application/x-gtar' => 'gtar',
'application/x-gzip' => 'gzip',
'application/mac-binhex40' => 'hqx',
'application/mac-binhex' => 'hqx',
'application/x-binhex40' => 'hqx',
'application/x-mac-binhex40' => 'hqx',
'text/html' => 'html',
'image/x-icon' => 'ico',
'image/x-ico' => 'ico',
'image/vnd.microsoft.icon' => 'ico',
'text/calendar' => 'ics',
'application/java-archive' => 'jar',
'application/x-java-application' => 'jar',
'application/x-jar' => 'jar',
'image/jp2' => 'jp2',
'video/mj2' => 'jp2',
'image/jpx' => 'jp2',
'image/jpm' => 'jp2',
'image/jpeg' => 'jpg',
'image/pjpeg' => 'jpg',
'application/x-javascript' => 'js',
'application/json' => 'json',
'text/json' => 'json',
'application/vnd.google-earth.kml+xml' => 'kml',
'application/vnd.google-earth.kmz' => 'kmz',
'text/x-log' => 'log',
'audio/x-m4a' => 'm4a',
'audio/mp4' => 'm4a',
'application/vnd.mpegurl' => 'm4u',
'audio/midi' => 'mid',
'application/vnd.mif' => 'mif',
'video/quicktime' => 'mov',
'video/x-sgi-movie' => 'movie',
'audio/mpeg' => 'mp3',
'audio/mpg' => 'mp3',
'audio/mpeg3' => 'mp3',
'audio/mp3' => 'mp3',
'video/mp4' => 'mp4',
'video/mpeg' => 'mpeg',
'application/oda' => 'oda',
'audio/ogg' => 'ogg',
'video/ogg' => 'ogg',
'application/ogg' => 'ogg',
'font/otf' => 'otf',
'application/x-pkcs10' => 'p10',
'application/pkcs10' => 'p10',
'application/x-pkcs12' => 'p12',
'application/x-pkcs7-signature' => 'p7a',
'application/pkcs7-mime' => 'p7c',
'application/x-pkcs7-mime' => 'p7c',
'application/x-pkcs7-certreqresp' => 'p7r',
'application/pkcs7-signature' => 'p7s',
'application/pdf' => 'pdf',
'application/octet-stream' => 'pdf',
'application/x-x509-user-cert' => 'pem',
'application/x-pem-file' => 'pem',
'application/pgp' => 'pgp',
'application/x-httpd-php' => 'php',
'application/php' => 'php',
'application/x-php' => 'php',
'text/php' => 'php',
'text/x-php' => 'php',
'application/x-httpd-php-source' => 'php',
'image/png' => 'png',
'image/x-png' => 'png',
'application/powerpoint' => 'ppt',
'application/vnd.ms-powerpoint' => 'ppt',
'application/vnd.ms-office' => 'ppt',
'application/msword' => 'ppt',
'application/vnd.openxmlformats-officedocument.presentationml.presentation' => 'pptx',
'application/x-photoshop' => 'psd',
'image/vnd.adobe.photoshop' => 'psd',
'audio/x-realaudio' => 'ra',
'audio/x-pn-realaudio' => 'ram',
'application/x-rar' => 'rar',
'application/rar' => 'rar',
'application/x-rar-compressed' => 'rar',
'audio/x-pn-realaudio-plugin' => 'rpm',
'application/x-pkcs7' => 'rsa',
'text/rtf' => 'rtf',
'text/richtext' => 'rtx',
'video/vnd.rn-realvideo' => 'rv',
'application/x-stuffit' => 'sit',
'application/smil' => 'smil',
'text/srt' => 'srt',
'image/svg+xml' => 'svg',
'application/x-shockwave-flash' => 'swf',
'application/x-tar' => 'tar',
'application/x-gzip-compressed' => 'tgz',
'image/tiff' => 'tiff',
'font/ttf' => 'ttf',
'text/plain' => 'txt',
'text/x-vcard' => 'vcf',
'application/videolan' => 'vlc',
'text/vtt' => 'vtt',
'audio/x-wav' => 'wav',
'audio/wave' => 'wav',
'audio/wav' => 'wav',
'application/wbxml' => 'wbxml',
'video/webm' => 'webm',
'image/webp' => 'webp',
'audio/x-ms-wma' => 'wma',
'application/wmlc' => 'wmlc',
'video/x-ms-wmv' => 'wmv',
'video/x-ms-asf' => 'wmv',
'font/woff' => 'woff',
'font/woff2' => 'woff2',
'application/xhtml+xml' => 'xhtml',
'application/excel' => 'xl',
'application/msexcel' => 'xls',
'application/x-msexcel' => 'xls',
'application/x-ms-excel' => 'xls',
'application/x-excel' => 'xls',
'application/x-dos_ms_excel' => 'xls',
'application/xls' => 'xls',
'application/x-xls' => 'xls',
'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet' => 'xlsx',
'application/vnd.ms-excel' => 'xlsx',
'application/xml' => 'xml',
'text/xml' => 'xml',
'text/xsl' => 'xsl',
'application/xspf+xml' => 'xspf',
'application/x-compress' => 'z',
'application/x-zip' => 'zip',
'application/zip' => 'zip',
'application/x-zip-compressed' => 'zip',
'application/s-compressed' => 'zip',
'multipart/x-zip' => 'zip',
'text/x-scriptzsh' => 'zsh'
];
expire_cached_files to DiskCache::expire()
2019-08-13 11:13:42 +00:00
public function __construct($dir) {
- fix multiple vulnerabilities in af_proxy_http - fix vulnerability in rewrite_relative_url() which prevented some URLs from being properly absolutized - fetch_file_contents: validate all URLs before requesting them - validate URLs: explicitly whitelist http and https scheme, forbid everything else - DiskCache/cached_url: only serve whitelisted content types (images, video) - simplify filename/URL handling code, remove and consolidate some less-used functions
2020-09-14 16:46:52 +00:00
$this->dir = CACHE_DIR . "/" . basename(clean($dir));
add classes/diskcache
2019-08-13 09:04:36 +00:00
}
expire_cached_files to DiskCache::expire()
2019-08-13 11:13:42 +00:00
public function getDir() {
add classes/diskcache
2019-08-13 09:04:36 +00:00
return $this->dir;
}
* HOOK_ENCLOSURE_ENTRY: pass article_id to handler * DiskCache: multiple fixes; support isWritable() for cache entries, set content-disposition for send() * public/cached_url: allow selecting files from sub-caches other than images * plugins/Cache_Starred_Images: rework to use DiskCache, can be enabled per-user, properly handles article enclosures, etc
2019-08-13 13:40:21 +00:00
public function makeDir() {
if (!is_dir($this->dir)) {
return mkdir($this->dir);
}
}
public function isWritable($filename = "") {
if ($filename) {
if (file_exists($this->getFullPath($filename)))
return is_writable($this->getFullPath($filename));
else
return is_writable($this->dir);
} else {
return is_writable($this->dir);
}
add DiskCache.isWritable
2019-08-13 09:15:43 +00:00
}
expire_cached_files to DiskCache::expire()
2019-08-13 11:13:42 +00:00
public function exists($filename) {
add classes/diskcache
2019-08-13 09:04:36 +00:00
return file_exists($this->getFullPath($filename));
}
expire_cached_files to DiskCache::expire()
2019-08-13 11:13:42 +00:00
public function getSize($filename) {
add classes/diskcache
2019-08-13 09:04:36 +00:00
if ($this->exists($filename))
return filesize($this->getFullPath($filename));
else
return -1;
}
expire_cached_files to DiskCache::expire()
2019-08-13 11:13:42 +00:00
public function getFullPath($filename) {
- fix multiple vulnerabilities in af_proxy_http - fix vulnerability in rewrite_relative_url() which prevented some URLs from being properly absolutized - fetch_file_contents: validate all URLs before requesting them - validate URLs: explicitly whitelist http and https scheme, forbid everything else - DiskCache/cached_url: only serve whitelisted content types (images, video) - simplify filename/URL handling code, remove and consolidate some less-used functions
2020-09-14 16:46:52 +00:00
return $this->dir . "/" . basename(clean($filename));
add classes/diskcache
2019-08-13 09:04:36 +00:00
}
expire_cached_files to DiskCache::expire()
2019-08-13 11:13:42 +00:00
public function put($filename, $data) {
add classes/diskcache
2019-08-13 09:04:36 +00:00
return file_put_contents($this->getFullPath($filename), $data);
}
expire_cached_files to DiskCache::expire()
2019-08-13 11:13:42 +00:00
public function touch($filename) {
add classes/diskcache
2019-08-13 09:04:36 +00:00
return touch($this->getFullPath($filename));
}
expire_cached_files to DiskCache::expire()
2019-08-13 11:13:42 +00:00
public function get($filename) {
add classes/diskcache
2019-08-13 09:04:36 +00:00
if ($this->exists($filename))
return file_get_contents($this->getFullPath($filename));
else
return null;
}
expire_cached_files to DiskCache::expire()
2019-08-13 11:13:42 +00:00
public function getMimeType($filename) {
add classes/diskcache
2019-08-13 09:04:36 +00:00
if ($this->exists($filename))
return mime_content_type($this->getFullPath($filename));
else
return null;
}
add DiskCache.send; switch af_zz_imgproxy to use DiskCache
2019-08-13 09:20:53 +00:00
DiskCache: append fake file extension when sending cached files based on mime type to make saving files easier
2020-05-12 10:28:54 +00:00
public function getFakeExtension($filename) {
$mimetype = $this->getMimeType($filename);
if ($mimetype)
return isset($this->mimeMap[$mimetype]) ? $this->mimeMap[$mimetype] : "";
else
return "";
}
expire_cached_files to DiskCache::expire()
2019-08-13 11:13:42 +00:00
public function send($filename) {
DiskCache: append fake file extension when sending cached files based on mime type to make saving files easier
2020-05-12 10:28:54 +00:00
$fake_extension = $this->getFakeExtension($filename);
if ($fake_extension)
$fake_extension = ".$fake_extension";
header("Content-Disposition: inline; filename=\"${filename}${fake_extension}\"");
* HOOK_ENCLOSURE_ENTRY: pass article_id to handler * DiskCache: multiple fixes; support isWritable() for cache entries, set content-disposition for send() * public/cached_url: allow selecting files from sub-caches other than images * plugins/Cache_Starred_Images: rework to use DiskCache, can be enabled per-user, properly handles article enclosures, etc
2019-08-13 13:40:21 +00:00
remove a lot of stuff from global context (functions.php), add a few helper classes instead
2020-09-22 06:04:33 +00:00
return $this->send_local_file($this->getFullPath($filename));
add DiskCache.send; switch af_zz_imgproxy to use DiskCache
2019-08-13 09:20:53 +00:00
}
add DiskCache.getUrl() and use it in a bunch of places
2019-08-13 09:39:21 +00:00
DiskCache: more strict checking for input filenames, getUrl() is no longer static
2019-08-14 06:49:18 +00:00
public function getUrl($filename) {
af_proxy_http: never print received data directly, always redirect to cached_url cache/getUrl: basename() passed filename just in case
2020-09-15 05:02:28 +00:00
return get_self_url_prefix() . "/public.php?op=cached_url&file=" . basename($this->dir) . "/" . basename($filename);
add DiskCache.getUrl() and use it in a bunch of places
2019-08-13 09:39:21 +00:00
}
move rewrite_cached_urls to DiskCache::rewriteUrls()
2019-08-13 09:46:57 +00:00
// check for locally cached (media) URLs and rewrite to local versions
// this is called separately after sanitize() and plugin render article hooks to allow
// plugins work on original source URLs used before caching
DiskCache: properly deal with srcset attributes
2020-04-29 16:29:36 +00:00
// NOTE: URLs should be already absolutized because this is called after sanitize()
move rewrite_cached_urls to DiskCache::rewriteUrls()
2019-08-13 09:46:57 +00:00
static public function rewriteUrls($str)
{
$res = trim($str);
if (!$res) return '';
$doc = new DOMDocument();
hide uninteresting errors in several DOMDocument->loadHTML() invocations
2020-10-01 10:20:07 +00:00
if (@$doc->loadHTML('<?xml encoding="UTF-8">' . $res)) {
move rewrite_cached_urls to DiskCache::rewriteUrls()
2019-08-13 09:46:57 +00:00
$xpath = new DOMXPath($doc);
$cache = new DiskCache("images");
DiskCache: properly deal with srcset attributes
2020-04-29 16:29:36 +00:00
$entries = $xpath->query('(//img[@src]|//source[@src|@srcset]|//video[@poster|@src])');
move rewrite_cached_urls to DiskCache::rewriteUrls()
2019-08-13 09:46:57 +00:00
$need_saving = false;
foreach ($entries as $entry) {
fix url rewriting for videos with poster and src if a poster attribute was present only that would have been rewritten and the (arguably more important) src attribute would be left as-is
2020-03-12 10:08:24 +00:00
foreach (array('src', 'poster') as $attr) {
if ($entry->hasAttribute($attr)) {
DiskCache: properly deal with srcset attributes
2020-04-29 16:29:36 +00:00
$url = $entry->getAttribute($attr);
$cached_filename = sha1($url);
move rewrite_cached_urls to DiskCache::rewriteUrls()
2019-08-13 09:46:57 +00:00
fix url rewriting for videos with poster and src if a poster attribute was present only that would have been rewritten and the (arguably more important) src attribute would be left as-is
2020-03-12 10:08:24 +00:00
if ($cache->exists($cached_filename)) {
DiskCache: properly deal with srcset attributes
2020-04-29 16:29:36 +00:00
$url = $cache->getUrl($cached_filename);
move rewrite_cached_urls to DiskCache::rewriteUrls()
2019-08-13 09:46:57 +00:00
DiskCache: properly deal with srcset attributes
2020-04-29 16:29:36 +00:00
$entry->setAttribute($attr, $url);
DiskCache::rewriteUrls() - remove img[@srcset]
2019-08-15 06:30:28 +00:00
$entry->removeAttribute("srcset");
move rewrite_cached_urls to DiskCache::rewriteUrls()
2019-08-13 09:46:57 +00:00
fix url rewriting for videos with poster and src if a poster attribute was present only that would have been rewritten and the (arguably more important) src attribute would be left as-is
2020-03-12 10:08:24 +00:00
$need_saving = true;
}
move rewrite_cached_urls to DiskCache::rewriteUrls()
2019-08-13 09:46:57 +00:00
}
}
DiskCache: properly deal with srcset attributes
2020-04-29 16:29:36 +00:00
if ($entry->hasAttribute("srcset")) {
better support for image srcset attributes as discussed in https://community.tt-rss.org/t/problem-with-img-srcset/3519
2020-06-15 08:58:59 +00:00
$matches = RSSUtils::decode_srcset($entry->getAttribute('srcset'));
DiskCache: properly deal with srcset attributes
2020-04-29 16:29:36 +00:00
better support for image srcset attributes as discussed in https://community.tt-rss.org/t/problem-with-img-srcset/3519
2020-06-15 08:58:59 +00:00
for ($i = 0; $i < count($matches); $i++) {
$cached_filename = sha1($matches[$i]["url"]);
DiskCache: properly deal with srcset attributes
2020-04-29 16:29:36 +00:00
if ($cache->exists($cached_filename)) {
better support for image srcset attributes as discussed in https://community.tt-rss.org/t/problem-with-img-srcset/3519
2020-06-15 08:58:59 +00:00
$matches[$i]["url"] = $cache->getUrl($cached_filename);
DiskCache: properly deal with srcset attributes
2020-04-29 16:29:36 +00:00
$need_saving = true;
}
}
better support for image srcset attributes as discussed in https://community.tt-rss.org/t/problem-with-img-srcset/3519
2020-06-15 08:58:59 +00:00
$entry->setAttribute("srcset", RSSUtils::encode_srcset($matches));
DiskCache: properly deal with srcset attributes
2020-04-29 16:29:36 +00:00
}
move rewrite_cached_urls to DiskCache::rewriteUrls()
2019-08-13 09:46:57 +00:00
}
if ($need_saving) {
fix some more warnings reported by phpstan
2021-02-06 14:38:24 +00:00
if (isset($doc->firstChild))
$doc->removeChild($doc->firstChild); //remove doctype
move rewrite_cached_urls to DiskCache::rewriteUrls()
2019-08-13 09:46:57 +00:00
$res = $doc->saveHTML();
}
}
return $res;
}
expire_cached_files to DiskCache::expire()
2019-08-13 11:13:42 +00:00
static function expire() {
$dirs = array_filter(glob(CACHE_DIR . "/*"), "is_dir");
foreach ($dirs as $cache_dir) {
$num_deleted = 0;
DiskCache::expire: support .no-auto-expiry to prevent automatic cache maintenance
2019-08-13 11:18:14 +00:00
if (is_writable($cache_dir) && !file_exists("$cache_dir/.no-auto-expiry")) {
expire_cached_files to DiskCache::expire()
2019-08-13 11:13:42 +00:00
$files = glob("$cache_dir/*");
if ($files) {
foreach ($files as $file) {
if (time() - filemtime($file) > 86400*CACHE_MAX_DAYS) {
unlink($file);
++$num_deleted;
}
}
}
DiskCache::expire: support .no-auto-expiry to prevent automatic cache maintenance
2019-08-13 11:18:14 +00:00
Debug::log("Expired $cache_dir: removed $num_deleted files.");
}
expire_cached_files to DiskCache::expire()
2019-08-13 11:13:42 +00:00
}
}
remove a lot of stuff from global context (functions.php), add a few helper classes instead
2020-09-22 06:04:33 +00:00
/* this is essentially a wrapper for readfile() which allows plugins to hook
output with httpd-specific "fast" implementation i.e. X-Sendfile or whatever else
hook function should return true if request was handled (or at least attempted to)
note that this can be called without user context so the plugin to handle this
should be loaded systemwide in config.php */
function send_local_file($filename) {
if (file_exists($filename)) {
if (is_writable($filename)) touch($filename);
$mimetype = mime_content_type($filename);
// this is hardly ideal but 1) only media is cached in images/ and 2) seemingly only mp4
// video files are detected as octet-stream by mime_content_type()
if ($mimetype == "application/octet-stream")
$mimetype = "video/mp4";
# block SVG because of possible embedded javascript (.....)
$mimetype_blacklist = [ "image/svg+xml" ];
/* only serve video and images */
fix some more warnings reported by phpstan
2021-02-06 14:38:24 +00:00
if (!preg_match("/(image|audio|video)\//", (string)$mimetype) || in_array($mimetype, $mimetype_blacklist)) {
remove a lot of stuff from global context (functions.php), add a few helper classes instead
2020-09-22 06:04:33 +00:00
http_response_code(400);
header("Content-type: text/plain");
print "Stored file has disallowed content type ($mimetype)";
return false;
}
$tmppluginhost = new PluginHost();
$tmppluginhost->load(PLUGINS, PluginHost::KIND_SYSTEM);
pluginhost: load plugin data automatically (also marks load_data method as private)
2021-01-15 05:32:06 +00:00
//$tmppluginhost->load_data();
remove a lot of stuff from global context (functions.php), add a few helper classes instead
2020-09-22 06:04:33 +00:00
foreach ($tmppluginhost->get_hooks(PluginHost::HOOK_SEND_LOCAL_FILE) as $plugin) {
if ($plugin->hook_send_local_file($filename)) return true;
}
header("Content-type: $mimetype");
fix some more warnings reported by phpstan
2021-02-06 14:38:24 +00:00
$stamp = gmdate("D, d M Y H:i:s", (int)filemtime($filename)) . " GMT";
remove a lot of stuff from global context (functions.php), add a few helper classes instead
2020-09-22 06:04:33 +00:00
header("Last-Modified: $stamp", true);
return readfile($filename);
} else {
return false;
}
}
add classes/diskcache
2019-08-13 09:04:36 +00:00
}