<?php

namespace spouts\rss;

/**
 * Plugin for fetching the news with fivefilters Full-Text RSS
 *
 * @copyright  Copyright (c) Tobias Zeising (http://www.aditu.de)
 * @license    GPLv3 (https://www.gnu.org/licenses/gpl-3.0.html)
 * @author     Tobias Zeising <tobias.zeising@aditu.de>
 */
class fulltextrss extends feed {
    /** @var string name of spout */
    public $name = 'RSS Feed (with FullTextRss)';

    /** @var string description of this source type */
    public $description = 'This feed extracts full text article from webpages with an embedded version of Full-Text RSS';

    /**
     * config params
     * array of arrays with name, type, default value, required, validation type
     *
     * - Values for type: text, password, checkbox, select
     * - Values for validation: alpha, email, numeric, int, alnum, notempty
     *
     * When type is "select", a new entry "values" must be supplied, holding
     * key/value pairs of internal names (key) and displayed labels (value).
     * See /spouts/rss/heise for an example.
     *
     * e.g.
     * array(
     *   "id" => array(
     *     "title"      => "URL",
     *     "type"       => "text",
     *     "default"    => "",
     *     "required"   => true,
     *     "validation" => array("alnum")
     *   ),
     *   ....
     * )
     *
     * @var bool|mixed
     */
    public $params = [
        'url' => [
            'title' => 'URL',
            'type' => 'url',
            'default' => '',
            'required' => true,
            'validation' => ['notempty']
        ],
    ];

    /** @var string tag for logger */
    public $tag = 'ftrss';

    private $extractor;

    private $fingerprints = [
        '<meta name="generator" content="Posterous"' => ['hostname' => 'fingerprint.posterous.com', 'head' => true],
        '<meta content=\'blogger\' name=\'generator\'' => ['hostname' => 'fingerprint.blogspot.com', 'head' => true],
        '<meta name="generator" content="Blogger"' => ['hostname' => 'fingerprint.blogspot.com', 'head' => true],
        // '<meta name="generator" content="WordPress.com"' => array('hostname'=>'fingerprint.wordpress.com', 'head'=>true),
        '<meta name="generator" content="WordPress' => ['hostname' => 'fingerprint.wordpress.com', 'head' => true]
    ];
    private $allowed_parsers = ['libxml', 'html5lib'];
    private $rewrite_relative_urls = true;

    /**
     * loads content for given source
     *
     * @param string $url
     *
     * @return void
     */
/*
    public function load($params) {
        parent::load(array( 'url' => $this->getXmlUrl($params)) );
    }
*/

    /**
     * returns the content of this item
     *
     * @return string content
     */
    public function getContent() {
        $url = parent::getLink();
        \F3::get('logger')->info($this->tag . ' - Loading page: ' . $url);
        $content = $this->fetchFromWebSite($url);
        if ($content === false) {
            \F3::get('logger')->error($this->tag . ' - Failed loading page');

            return parent::getContent() .
                '<p><strong>Failed to get web page</strong></p>';
        }

        \F3::get('logger')->info($this->tag . ' - Extracting content');
        $content = @$this->extractContent($content, parent::getLink());
        if ($content === false) {
            \F3::get('logger')->error($this->tag . ' - Failed extracting content');

            return parent::getContent() .
                '<p><strong>Full Text RSS extracting error</strong></p>';
        }

        \F3::get('logger')->info($this->tag . ' - Cleaning content');
        $content = $this->cleanContent($content);
        if ($content === false) {
            \F3::get('logger')->error($this->tag . ' - Failed cleaning content from');

            return parent::getContent() .
                '<p><strong>Full Text RSS cleaning error</strong></p>';
        }

        return $content;
    }

    /**
     * fetch content from FullTextRss
     *
     * @author Jean Baptiste Favre
     *
     * @return string content
     */
    private function fetchFromWebSite($url) {
        $this->extractor = new \ContentExtractor(\F3::get('FTRSS_DATA_DIR') . '/custom', \F3::get('FTRSS_DATA_DIR') . '/standard');
        if (\F3::get('logger_level') === 'DEBUG') {
            ob_start();
            $this->extractor->debug = true;
            \SiteConfig::$debug = true;
        }
        \SiteConfig::use_apc(false);
        $this->extractor->fingerprints = $this->fingerprints;
        $this->extractor->allowedParsers = $this->allowed_parsers;

        $stream_opts = [
            'http' => [
                'timeout' => 5,
                'method' => 'GET',
                'header' => [
                    'Accept-language: en-us,en-gb;q=0.8,en;q=0.6,fr;q=0.4,fr-fr;q=0.2',
                    'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
                    'User-Agent: SimplePie/1.3.1 (Feed Parser; http://simplepie.org; Allow like Gecko) Build/20121030175911',
                    'DNT: 1',
                ]
          ]
        ];
        $context = stream_context_create($stream_opts);

        $url = $this->removeTrackersFromUrl($url);

        // Load web page
        $html = @file_get_contents($url, false, $context);
        if ($html === false) {
            return false;
        }

        return $html;
    }

    /**
     * remove tarkers from url
     *
     * @author Jean Baptiste Favre
     *
     * @param string $url
     *
     * @return string url
     */
    private function removeTrackersFromUrl($url) {
        $url = parse_url($url);

        // Next, rebuild URL
        $real_url = $url['scheme'] . '://';
        if (isset($url['user']) && isset($url['password'])) {
            $real_url .= $url['user'] . ':' . $url['password'] . '@';
        }
        $real_url .= $url['host'] . $url['path'];

        // Query string
        if (isset($url['query'])) {
            parse_str($url['query'], $q_array);
            $real_query = [];
            foreach ($q_array as $key => $value) {
                // Remove utm_* parameters
                if (strpos($key, 'utm_') === false) {
                    $real_query[] = $key . '=' . $value;
                }
            }
            $real_url .= '?' . implode('&', $real_query);
        }
        // Fragment
        if (isset($url['fragment'])) {
            // Remove xtor=RSS anchor
            if (strpos($url['fragment'], 'xtor=RSS') === false) {
                $real_url .= '#' . $url['fragment'];
            }
        }

        return $real_url;
    }

    /**
     * Extract full text from a full web page
     * I supress all Notices & Warnings of ContentExtractor & Readability
     * for ensuring working plugin in PHP Strict mode
     *
     * @author Jean Baptiste Favre
     *
     * @param string $html
     *
     * @return string html
     */
    private function extractContent($html, $url) {
        \F3::get('logger')->debug($this->tag . ' - Cleaning content');
        // remove strange things
        $html = str_replace('</[>', '', $html);
        $html = $this->convert_to_utf8($html, $response['headers']);

        $extract_result = $this->extractor->process($html, $url);
        if ($extract_result === false) {
            return false;
        }

        $extracted_content = @$this->extractor->getContent();
        $readability = @$this->extractor->readability;
        if (\F3::get('logger_level') === 'DEBUG') {
            $readability->debug = true;
        }
        @$readability->clean($extracted_content, 'select');
        if ($this->rewrite_relative_urls) {
            $this->makeAbsolute($url, $extracted_content);
        }
        unset($readability);

        // remove nesting: <div><div><div><p>test</p></div></div></div> = <p>test</p>
        while ($extracted_content->childNodes->length == 1 && $extracted_content->firstChild->nodeType === XML_ELEMENT_NODE) {
            // only follow these tag names
            if (!in_array(strtolower($extracted_content->tagName), ['div', 'article', 'section', 'header', 'footer'])) {
                break;
            }
            $extracted_content = $extracted_content->firstChild;
        }

        // Need to preserve things like body: //img[@id='feature']
        if (in_array(strtolower($extracted_content->tagName), ['div', 'article', 'section', 'header', 'footer'])) {
            $html = $extracted_content->innerHTML;
        } else {
            $html = @$extracted_content->ownerDocument->saveXML($extracted_content); // essentially outerHTML
        }
        unset($extracted_content);

        return $html;
    }

    /**
     * Clean extracted content before giving it back to SelfOSS
     *
     * @author Jean Baptiste Favre
     *
     * @param string $html
     *
     * @return string html
     */
    private function cleanContent($html) {
        // post-processing cleanup
        \F3::get('logger')->debug($this->tag . ' - Post process cleaning & anti-XSS');
        $html = preg_replace('!<p>[\s\h\v]*</p>!u', '', $html);
        $html = preg_replace('!<a[^>]*/>!', '', $html);

        if ($html === false) {
            return $false;
        }

        return $html;
    }

    private function convert_to_utf8($html, $header = null) {
        $encoding = null;
        if ($html || $header) {
            if (is_array($header)) {
                $header = implode("\n", $header);
            }
            if (!$header || !preg_match_all('/^Content-Type:\s+([^;]+)(?:;\s*charset=["\']?([^;"\'\n]*))?/im', $header, $match, PREG_SET_ORDER)) {
                // error parsing the response
                \F3::get('logger')->debug($this->tag . ' - Could not find Content-Type header in HTTP response');
            } else {
                $match = end($match); // get last matched element (in case of redirects)
                if (isset($match[2])) {
                    $encoding = trim($match[2], "\"' \r\n\0\x0B\t");
                }
            }
            // TODO: check to see if encoding is supported (can we convert it?)
            // If it's not, result will be empty string.
            // For now we'll check for invalid encoding types returned by some sites, e.g. 'none'
            // Problem URL: http://facta.co.jp/blog/archives/20111026001026.html
            if (!$encoding || $encoding == 'none') {
                // search for encoding in HTML - only look at the first 50000 characters
                // Why 50000? See, for example, http://www.lemonde.fr/festival-de-cannes/article/2012/05/23/deux-cretes-en-goguette-sur-la-croisette_1705732_766360.html
                // TODO: improve this so it looks at smaller chunks first
                $html_head = substr($html, 0, 50000);
                if (preg_match('/^<\?xml\s+version=(?:"[^"]*"|\'[^\']*\')\s+encoding=("[^"]*"|\'[^\']*\')/s', $html_head, $match)) {
                    $encoding = trim($match[1], '"\'');
                } elseif (preg_match('/<meta\s+http-equiv=["\']?Content-Type["\']? content=["\'][^;]+;\s*charset=["\']?([^;"\'>]+)/i', $html_head, $match)) {
                    $encoding = trim($match[1]);
                } elseif (preg_match_all('/<meta\s+([^>]+)>/i', $html_head, $match)) {
                    foreach ($match[1] as $_test) {
                        if (preg_match('/charset=["\']?([^"\']+)/i', $_test, $_m)) {
                            $encoding = trim($_m[1]);
                            break;
                        }
                    }
                }
            }
            if (isset($encoding)) {
                $encoding = trim($encoding);
            }
            // trim is important here!
            if (!$encoding || (strtolower($encoding) == 'iso-8859-1')) {
                // replace MS Word smart qutoes
                $trans = [];
                $trans[chr(130)] = '&sbquo;';    // Single Low-9 Quotation Mark
                $trans[chr(131)] = '&fnof;';    // Latin Small Letter F With Hook
                $trans[chr(132)] = '&bdquo;';    // Double Low-9 Quotation Mark
                $trans[chr(133)] = '&hellip;';    // Horizontal Ellipsis
                $trans[chr(134)] = '&dagger;';    // Dagger
                $trans[chr(135)] = '&Dagger;';    // Double Dagger
                $trans[chr(136)] = '&circ;';    // Modifier Letter Circumflex Accent
                $trans[chr(137)] = '&permil;';    // Per Mille Sign
                $trans[chr(138)] = '&Scaron;';    // Latin Capital Letter S With Caron
                $trans[chr(139)] = '&lsaquo;';    // Single Left-Pointing Angle Quotation Mark
                $trans[chr(140)] = '&OElig;';    // Latin Capital Ligature OE
                $trans[chr(145)] = '&lsquo;';    // Left Single Quotation Mark
                $trans[chr(146)] = '&rsquo;';    // Right Single Quotation Mark
                $trans[chr(147)] = '&ldquo;';    // Left Double Quotation Mark
                $trans[chr(148)] = '&rdquo;';    // Right Double Quotation Mark
                $trans[chr(149)] = '&bull;';    // Bullet
                $trans[chr(150)] = '&ndash;';    // En Dash
                $trans[chr(151)] = '&mdash;';    // Em Dash
                $trans[chr(152)] = '&tilde;';    // Small Tilde
                $trans[chr(153)] = '&trade;';    // Trade Mark Sign
                $trans[chr(154)] = '&scaron;';    // Latin Small Letter S With Caron
                $trans[chr(155)] = '&rsaquo;';    // Single Right-Pointing Angle Quotation Mark
                $trans[chr(156)] = '&oelig;';    // Latin Small Ligature OE
                $trans[chr(159)] = '&Yuml;';    // Latin Capital Letter Y With Diaeresis
                $html = strtr($html, $trans);
            }
            if (!$encoding) {
                \F3::get('logger')->debug($this->tag . ' - No character encoding found, so treating as UTF-8');
                $encoding = 'utf-8';
            } else {
                \F3::get('logger')->debug($this->tag . ' - Character encoding: ' . $encoding);
                if (strtolower($encoding) != 'utf-8') {
                    //('Converting to UTF-8');
                    $html = \SimplePie_Misc::change_encoding($html, $encoding, 'utf-8');
                }
            }
        }

        return $html;
    }

    private function makeAbsolute($base, $elem) {
        $base = new \SimplePie_IRI($base);
        // remove '//' in URL path (used to prevent URLs from resolving properly)
        // TODO: check if this is still the case
        if (isset($base->path)) {
            $base->path = preg_replace('!//+!', '/', $base->path);
        }
        foreach (['a' => 'href', 'img' => 'src'] as $tag => $attr) {
            $elems = $elem->getElementsByTagName($tag);
            for ($i = $elems->length - 1; $i >= 0; --$i) {
                $e = $elems->item($i);
                $this->makeAbsoluteAttr($base, $e, $attr);
            }
            if (strtolower($elem->tagName) == $tag) {
                makeAbsoluteAttr($base, $elem, $attr);
            }
        }
    }

    private function makeAbsoluteAttr($base, $e, $attr) {
        if ($e->hasAttribute($attr)) {
            // Trim leading and trailing white space. I don't really like this but
            // unfortunately it does appear on some sites. e.g.  <img src=" /path/to/image.jpg" />
            $url = trim(str_replace('%20', ' ', $e->getAttribute($attr)));
            $url = str_replace(' ', '%20', $url);
            if (!preg_match('!https?://!i', $url)) {
                if ($absolute = \SimplePie_IRI::absolutize($base, $url)) {
                    $e->setAttribute($attr, $absolute);
                }
            }
        }
    }
}
