diff options
145 files changed, 9812 insertions, 2906 deletions
diff --git a/.dockerignore b/.dockerignore index 15154f0..f2bc0e8 100644 --- a/.dockerignore +++ b/.dockerignore @@ -4,5 +4,4 @@ DEBUG Dockerfile whitelist.txt phpcs.xml -CHANGELOG.md CONTRIBUTING.md
\ No newline at end of file diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md new file mode 100644 index 0000000..18fc93f --- /dev/null +++ b/.github/CONTRIBUTING.md @@ -0,0 +1,47 @@ +### Pull request policy + +* [Fix one issue per pull request](https://github.com/RSS-Bridge/rss-bridge/wiki/Pull-request-policy#fix-one-issue-per-pull-request) +* [Respect the coding style policy](https://github.com/RSS-Bridge/rss-bridge/wiki/Pull-request-policy#respect-the-coding-style-policy) +* [Properly name your commits](https://github.com/RSS-Bridge/rss-bridge/wiki/Pull-request-policy#properly-name-your-commits) + * When fixing a bridge (located in the `bridges` directory), write `[BridgeName] Feature` <br>(i.e. `[YoutubeBridge] Fix typo in video titles`). + * When fixing other files, use `[FileName] Feature` <br>(i.e. `[index.php] Add multilingual support`). + * When fixing a general problem that applies to multiple files, write `category: feature` <br>(i.e. `bridges: Fix various typos`). + +Note that all pull-requests must pass all tests before they can be merged. + +### Coding style + +* [Whitespace](https://github.com/RSS-Bridge/rss-bridge/wiki/Whitespace) + * [Add a new line at the end of a file](https://github.com/RSS-Bridge/rss-bridge/wiki/Whitespace#add-a-new-line-at-the-end-of-a-file) + * [Do not add a whitespace before a semicolon](https://github.com/RSS-Bridge/rss-bridge/wiki/Whitespace#add-a-new-line-at-the-end-of-a-file) + * [Do not add whitespace at start or end of a file or end of a line](https://github.com/RSS-Bridge/rss-bridge/wiki/Whitespace#do-not-add-whitespace-at-start-or-end-of-a-file-or-end-of-a-line) +* [Indentation](https://github.com/RSS-Bridge/rss-bridge/wiki/Indentation) + * [Use tabs for indentation](https://github.com/RSS-Bridge/rss-bridge/wiki/Indentation#use-tabs-for-indentation) +* [Maximum line length](https://github.com/RSS-Bridge/rss-bridge/wiki/Maximum-line-length) + * [The maximum line length should not exceed 80 characters](https://github.com/RSS-Bridge/rss-bridge/wiki/Maximum-line-length#the-maximum-line-length-should-not-exceed-80-characters) +* [Strings](https://github.com/RSS-Bridge/rss-bridge/wiki/Strings) + * [Whenever possible use single quoted strings](https://github.com/RSS-Bridge/rss-bridge/wiki/Strings#whenever-possible-use-single-quote-strings) + * [Add spaces around the concatenation operator](https://github.com/RSS-Bridge/rss-bridge/wiki/Strings#add-spaces-around-the-concatenation-operator) + * [Use a single string instead of concatenating](https://github.com/RSS-Bridge/rss-bridge/wiki/Strings#use-a-single-string-instead-of-concatenating) +* [Constants](https://github.com/RSS-Bridge/rss-bridge/wiki/Constants) + * [Use UPPERCASE for constants](https://github.com/RSS-Bridge/rss-bridge/wiki/Constants#use-uppercase-for-constants) +* [Keywords](https://github.com/RSS-Bridge/rss-bridge/wiki/Keywords) + * [Use lowercase for `true`, `false` and `null`](https://github.com/RSS-Bridge/rss-bridge/wiki/Keywords#use-lowercase-for-true-false-and-null) +* [Operators](https://github.com/RSS-Bridge/rss-bridge/wiki/Operators) + * [Operators must have a space around them](https://github.com/RSS-Bridge/rss-bridge/wiki/Operators#operators-must-have-a-space-around-them) +* [Functions](https://github.com/RSS-Bridge/rss-bridge/wiki/Functions) + * [Parameters with default values must appear last in functions](https://github.com/RSS-Bridge/rss-bridge/wiki/Functions#parameters-with-default-values-must-appear-last-in-functions) + * [Calling functions](https://github.com/RSS-Bridge/rss-bridge/wiki/Functions#calling-functions) + * [Do not add spaces after opening or before closing bracket](https://github.com/RSS-Bridge/rss-bridge/wiki/Functions#do-not-add-spaces-after-opening-or-before-closing-bracket) +* [Structures](https://github.com/RSS-Bridge/rss-bridge/wiki/Structures) + * [Structures must always be formatted as multi-line blocks](https://github.com/RSS-Bridge/rss-bridge/wiki/Structures#structures-must-always-be-formatted-as-multi-line-blocks) +* [If-Statement](https://github.com/RSS-Bridge/rss-bridge/wiki/if-Statement) + * [Use `elseif` instead of `else if`](https://github.com/RSS-Bridge/rss-bridge/wiki/if-Statement#use-elseif-instead-of-else-if) + * [Do not write empty statements](https://github.com/RSS-Bridge/rss-bridge/wiki/if-Statement#do-not-write-empty-statements) + * [Do not write unconditional if-statements](https://github.com/RSS-Bridge/rss-bridge/wiki/if-Statement#do-not-write-unconditional-if-statements) +* [Classes](https://github.com/RSS-Bridge/rss-bridge/wiki/Classes) + * [Use PascalCase for class names](https://github.com/RSS-Bridge/rss-bridge/wiki/Classes#use-pascalcase-for-class-names) + * [Do not use final statements inside final classes](https://github.com/RSS-Bridge/rss-bridge/wiki/Classes#do-not-use-final-statements-inside-final-classes) + * [Do not override methods to call their parent](https://github.com/RSS-Bridge/rss-bridge/wiki/Classes#do-not-override-methods-to-call-their-parent) +* [Casting](https://github.com/RSS-Bridge/rss-bridge/wiki/Casting) + * [Do not add spaces when casting](https://github.com/RSS-Bridge/rss-bridge/wiki/Casting#do-not-add-spaces-when-casting) diff --git a/.github/ISSUE_TEMPLATE/bridge-request-template.md b/.github/ISSUE_TEMPLATE/bridge-request-template.md new file mode 100644 index 0000000..f4b1119 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bridge-request-template.md @@ -0,0 +1,61 @@ +--- +name: Bridge request template +about: Use this template for requesting a new bridge + +--- + +# Bridge request + +<!-- +This is a bridge request. Start by adding a descriptive title (i.e. `Bridge request for GitHub`). Use the "Preview" button to see a preview of your request. Make sure your request is complete before submitting! + +Notice: This comment is only visible to you while you work on your request. Please do not remove any of the lines in the template (you may add your own outside the "<!--" and "- ->" lines!) +--> + +## General information + +<!-- +Please describe what you expect from the bridge. Whenever possible provide sample links and screenshots (you can just paste them here) to express your expectations and help others understand your request. If possible, mark relevant areas in your screenshot. Use the following questions for reference: +--> + +- _Host URI for the bridge_ (i.e. `https://github.com`): + +- Which information would you like to see? + + + +- How should the information be displayed/formatted? + + + +- Which of the following parameters do you expect? + + - [X] Title + - [X] URI (link to the original article) + - [ ] Author + - [ ] Timestamp + - [X] Content (the content of the article) + - [ ] Enclosures (pictures, videos, etc...) + - [ ] Categories (categories, tags, etc...) + +## Options + +<!--Select options from the list below. Add your own option if one is missing:--> + +- [ ] Limit number of returned items + - _Default limit_: 5 +- [ ] Load full articles + - _Cache articles_ (articles are stored in a local cache on first request): yes + - _Cache timeout_ (max = 24 hours): 24 hours +- [X] Balance requests (RSS-Bridge uses cached versions to reduce bandwith usage) + - _Timeout_ (default = 5 minutes, max = 24 hours): 5 minutes + +<!--Be aware that some options might not be available for your specific request due to technical limitations!--> + +<!-- +## Additional notes + +Keep in mind that opening a request does not guarantee the bridge being implemented! That depends entirely on the interest and time of others to make the bridge for you. + +You can also implement your own bridge (with support of the community if needed). Find more information in the [RSS-Bridge Wiki](https://github.com/RSS-Bridge/rss-bridge/wiki/For-developers) developer section. +--> diff --git a/.travis.yml b/.travis.yml index cd5e2d9..80f141b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,12 +3,35 @@ sudo: false language: php install: - - pear channel-update pear.php.net - - pear install PHP_CodeSniffer + - composer global require dealerdirect/phpcodesniffer-composer-installer; + - composer global require phpcompatibility/php-compatibility; + # Use PHPUnit 6 for unit tests (stable), requires PHP 7 + - if [[ $TRAVIS_PHP_VERSION == "7.0" ]]; then + composer global require phpunit/phpunit ^6; + fi + # Use latest PHPUnit on nightly to detect breaking changes + - if [[ $TRAVIS_PHP_VERSION == "nightly" ]]; then + composer global require phpunit/phpunit; + fi script: - phpenv rehash - - phpcs . --standard=phpcs.xml --warning-severity=0 --extensions=php -p + # Run PHP_CodeSniffer on all versions + - ~/.composer/vendor/bin/phpcs . --standard=phpcs.xml --warning-severity=0 --extensions=php -p; + # Check PHP compatibility for the lowest supported version + - if [[ $TRAVIS_PHP_VERSION == "5.6" ]]; then + ~/.composer/vendor/bin/phpcs . --standard=phpcompatibility.xml --warning-severity=0 --extensions=php -p; + fi + # Run unit tests (stable) + - if [[ $TRAVIS_PHP_VERSION == "7.0" ]]; then + phpunit --configuration=phpunit.xml --include-path=lib/; + fi + # Run unit tests (latest/nightly) + # Check PHP compatibility for all versions, starting at the lowest supported version in order to detect breaking changes + - if [[ $TRAVIS_PHP_VERSION == "nightly" ]]; then + phpunit --configuration=phpunit.xml --include-path=lib/; + ~/.composer/vendor/bin/phpcs . --standard=PHPCompatibility --warning-severity=0 --extensions=php -p --runtime-set testVersion 5.6-; + fi matrix: fast_finish: true @@ -16,9 +39,7 @@ matrix: include: - php: 5.6 - php: 7.0 - - php: hhvm - php: nightly allow_failures: - - php: hhvm - php: nightly diff --git a/CHANGELOG.md b/CHANGELOG.md deleted file mode 100644 index 467040e..0000000 --- a/CHANGELOG.md +++ /dev/null @@ -1,263 +0,0 @@ -rss-bridge Changelog -=== - -RSS-Bridge 2017-08-19 -== - -## General changes -* whitelist: Do case-insensitive whitelist matching -* [FeedExpander] Fix Serialization of 'SimpleXMLElement' is not allowed -* [FeedExpander] Remove whitespace from source content -* [index] Add GET parameter 'q' for search queries - - **Example**: You can now add `&q=Twitter` to load into the search field -* [index] Check permissions for cache folder and whitelist file -* [index] Show bridge options when loading with URL fragment - - **Example**: You can now add `#bridge-Twitter` to load the card with all -parameters visible -* [style] Center search cursor and hide placeholder -* [validation] Fix error on undefined optional numeric value - -## Modified bridges -* [DanbooruBridge] Allow descendant classes to override tag collection -* [DribbbleBridge] Add dribble bridge listing last dribble popular shots (#558) -* [FacebookBridge] Fix & in URLs -* [GelbooruBridge] Fix bridge not getting tags correctly -* [GoComicsBridge] Fix for page structure changes (#568) -* [LeBonCoinBridge] Fix bridge is marked executable -* [LWNprevBridge] Fix everchanging url -* [YoutubeBridge] Fix error on certain keywords -* [YoutubeBridge] Fix issues loading playlists - -## Removed bridges -* VineBridge - -RSS-Bridge 2017-08-03 -== - -## Important changes -* RSS-Bridge now has [contribution guidelines](CONTRIBUTING.md) -* [phpcs rules](phpcs.xml) follow the [contribution guidelines](CONTRIBUTING.md) - -## General changes -* Added a search bar to make searching for bridges easier -* Added user friendly error page for when a bridge fails -* Added caching of extraInfos (name, uri) -* Added an indicator to warn for bridges using HTTP instead of HTTPS -* Various bug fixes and improvements - -## Modified bridges -* AllocineFRBridge] Update Faux Raccord link -* [DanbooruBridge] Fix broken URI -* [DuckDuckGoBridge] Disable DuckDuckGo redirects so that the links returned are correct. -* [FacebookBridge] Add option to hide posts with facebook videos -* [FacebookBridge] Add requester languages to HTTP header -* [FacebookBridge] Handle summary posts -* [FacebookBridge] Replace 'novideo' with 'media_type' -* [FilterBridge] Initial implementation of basic title permit and block -* [FlickrTagBridge] Fix and improve bridge by using the FlickrExploreBridge approach -* [GooglePlusPostBridge] Autofix user names -* [GooglePlusPostBridge] Fix bridge implementation -* [GooglePlusPostBridge] Fix content loading -* [InstagramBridge] Add option to filter for videos and pictures -* [LWNprevBridge] full rewrite -* [MangareaderBridge] Fix double forward slashes -* [NasaApodBridge] Use HTTPS instead of HTTP -* [PinterestBridge] Fix checkbox not working -* [PinterestBridge] Fix implementation after DOM changes -* [RTBFBridge] Update URI -* [SexactuBridge] Fix URI and timestamp -* [SexactuBridge] Use most modern version of bridge api and cached pages (#504) -* [ShanaprojectBridge] Don't throw error if timestamp is missing -* [TwitterBridge] Add option to hide retweets -* [TwitterBridge] Avoid empty content caused by new login policy -* [TwitterBridge] Fix double slashes in URI -* [TwitterBridge] Fix missing spaces -* [TwitterBridge] Fix title includes anchors in plaintext format -* [TwitterBridge] ignore promoted tweets -* [TwitterBridge] Optimize returned image sizes -* [TwitterBridge] Show quotes and pictures -* [WebfailBridge] Properly handle gifs (DOM changed) -* [YoutubeBridge] Improve readability of feed contents -* [YoutubeBridge] Improve URL handling in video descriptions - -## New bridges -* AmazonBridge -* DiceBridge -* EtsyBridge -* FB2Bridge -* FilterBridge -* FlickrBridge -* GithubSearchBridge -* GoComicsBridge -* KATBridge -* KernelBugTrackerBridge -* MixCloudBridge -* MoinMoinBridge -* RainbowSixSiegeBridge -* SteamBridge -* TheTVDBBridge -* Torrent9Bridge -* UsbekEtRicaBridge -* WikiLeaksBridge -* WordPressPluginUpdateBridge - -Alpha 0.2 -=== - -## Important changes -* RSS-Bridge has been [UNLICENSED](UNLICENSE) -* RSS-Bridge is now a community-managed project on [GitHub](https://github.com/rss-bridge/rss-bridge) -* RSS-Bridge now has a [Wiki](https://github.com/rss-bridge/rss-bridge/wiki) -* RSS-Bridge now supports [Travis-CI](https://travis-ci.org) - -## General changes -* Added [CHANGELOG](CHANGELOG.md) (this file) -* Added [PHP Simple HTML DOM Parser](http://simplehtmldom.sourceforge.net) to [vendor](vendor/simplehtmldom/) -* Added cache purging function (cache will be force-purged after 24 hours or as defined by bridge) -* Added new format [MrssFormat](formats/MrssFormat.php) -* Added parameter `author` - for display of the feed author name - to all formats -* Added new abstraction of the BridgeInterface: - - [FeedExpander](https://github.com/RSS-Bridge/rss-bridge/wiki/Bridge-API) -* Added optional support for proxy usage on each individual bridge -* Added support for [custom bridge parameter](https://github.com/RSS-Bridge/rss-bridge/wiki/BridgeAbstract#format-specifications) (text, number, list, checkbox) -* Changed design of the welcome screen -* Changed design of HtmlFormat -* Changed behavior of debug mode: - - Enable debug mode by placing a file called "DEBUG" in the root folder - - Debug mode automatically disables cache file loading -* Changed implementation of bridges - see [Wiki](https://github.com/rss-bridge/rss-bridge/wiki) - - Changed comment-style metadata to constants - - Added support for multiple utilizations per bridge - - Changed the parameter loading algorithm to be loaded by RSS-Bridge core -* Improved checks for PHP version, configuration and extensions -* Many bug fixes - -## Modified Bridges -* FlickrExploreBridge -* GoogleSearchBridge -* TwitterBridge - -## New Bridges -* ABCTabsBridge -* AcrimedBridge -* AllocineFRBridge -* AnimeUltimeBridge -* Arte7Bridge -* AskfmBridge -* BandcampBridge -* BastaBridge -* BlaguesDeMerdeBridge -* BooruprojectBridge -* CADBridge -* CNETBridge -* CastorusBridge -* CollegeDeFranceBridge -* CommonDreamsBridge -* CopieDoubleBridge -* CourrierInternationalBridge -* CpasbienBridge -* CryptomeBridge -* DailymotionBridge -* DanbooruBridge -* DansTonChatBridge -* DauphineLibereBridge -* DemoBridge -* DeveloppezDotComBridge -* DilbertBridge -* DollbooruBridge -* DuckDuckGoBridge -* EZTVBridge -* EliteDangerousGalnetBridge -* ElsevierBridge -* EstCeQuonMetEnProdBridge -* FacebookBridge -* FierPandaBridge -* FlickrTagBridge -* FootitoBridge -* FourchanBridge -* FuturaSciencesBridge -* GBAtempBridge -* GelbooruBridge -* GiphyBridge -* GithubIssueBridge -* GizmodoBridge -* GooglePlusPostBridge -* HDWallpapersBridge -* HentaiHavenBridge -* IdenticaBridge -* InstagramBridge -* IsoHuntBridge -* JapanExpoBridge -* KonachanBridge -* KoreusBridge -* KununuBridge -* LWNprevBridge -* LeBonCoinBridge -* LegifranceJOBridge -* LeMondeInformatiqueBridge -* LesJoiesDuCodeBridge -* LichessBridge -* LinkedInCompanyBridge -* LolibooruBridge -* MangareaderBridge -* MilbooruBridge -* MoebooruBridge -* MondeDiploBridge -* MsnMondeBridge -* MspabooruBridge -* NasaApodBridge -* NeuviemeArtBridge -* NextInpactBridge -* NextgovBridge -* NiceMatinBridge -* NovelUpdatesBridge -* OpenClassroomsBridge -* ParuVenduImmoBridge -* PickyWallpapersBridge -* PinterestBridge -* PlanetLibreBridge -* RTBFBridge -* ReadComicsBridge -* Releases3DSBridge -* ReporterreBridge -* Rue89Bridge -* Rule34Bridge -* Rule34pahealBridge -* SafebooruBridge -* SakugabooruBridge -* ScmbBridge -* ScoopItBridge -* SensCritiqueBridge -* SexactuBridge -* ShanaprojectBridge -* Shimmie2Bridge -* SoundcloudBridge -* StripeAPIChangeLogBridge -* SuperbWallpapersBridge -* T411Bridge -* TagBoardBridge -* TbibBridge -* TheCodingLoveBridge -* TheHackerNewsBridge -* ThePirateBayBridge -* UnsplashBridge -* ViadeoCompanyBridge -* VineBridge -* VkBridge -* WallpaperStopBridge -* WebfailBridge -* WeLiveSecurityBridge -* WhydBridge -* WikipediaBridge -* WordPressBridge -* WorldOfTanksBridge -* XbooruBridge -* YandereBridge -* YoutubeBridge -* ZDNetBridge - -Alpha 0.1 -=== -* First tagged version. -* Includes refactoring. -* Unstable.
\ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md deleted file mode 100644 index e03f926..0000000 --- a/CONTRIBUTING.md +++ /dev/null @@ -1,47 +0,0 @@ -### Pull request policy -Fix one issue per pull request. -Squash commits before opening a pull request. -Respect the coding style policy. -Name your PR like the following : - -* When correcting a single bridge, use `[BridgeName] Feature`. -* When fixing a problem in a specific file, use `[FileName] Feature`. -* When fixing a general problem, use `category : feature`. - -Note that all pull-requests should pass the unit tests before they can be merged. - -### Coding style - -Use `camelCase` for variables and methods. -Use `UPPERCASE` for constants. -Use `PascalCase` for class names. When creating a bridge, your class and PHP file should be named `MyImplementationBridge`. -Use tabs for indentation. -Add an empty line at the end of your file. - -Use `''` to encapsulate strings, including in arrays. -Prefer lines shorter than 80 chars, no line longer than 120 chars. -PHP constants should be in lower case (`true, false, null`...) - - -* Add spaces between the logical operator and your expressions (not needed for the `!` operator). -* Use `||` and `&&` instead of `or` and `and`. -* Add space between your condition and the opening bracket/closing bracket. -* Don't put a space between `if` and your bracket. -* Use `elseif` instead of `else if`. -* Add new lines in your conditions if they are containing more than one line. -* Example : - -```PHP -if($a == true && $b) { - print($a); -} else if(!$b) { - - $a = !$a; - $b = $b >> $a; - print($b); - -} else { - print($b); -} -``` - @@ -1,10 +1,12 @@ rss-bridge === -[](UNLICENSE) [](https://travis-ci.org/RSS-Bridge/rss-bridge) +[](UNLICENSE) [](https://github.com/rss-bridge/rss-bridge/releases/latest) [](https://tracker.debian.org/pkg/rss-bridge) [](https://www.gnu.org/software/guix/packages/R/) [](https://travis-ci.org/RSS-Bridge/rss-bridge) [](https://hub.docker.com/r/rssbridge/rss-bridge/) -rss-bridge is a PHP project capable of generating ATOM feeds for websites which don't have one. +RSS-Bridge is a PHP project capable of generating RSS and Atom feeds for websites which don't have one. It can be used on webservers or as stand alone application in CLI mode. -Supported sites/pages (main) +**Important**: RSS-Bridge is __not__ a feed reader or feed aggregator, but a tool to generate feeds that are consumed by feed readers and feed aggregators. Find a list of feed aggregators on [Wikipedia](https://en.wikipedia.org/wiki/Comparison_of_feed_aggregators). + +Supported sites/pages (examples) === * `Bandcamp` : Returns last release from [bandcamp](https://bandcamp.com/) for a tag @@ -25,106 +27,188 @@ Supported sites/pages (main) * `Wikipedia`: highlighted articles from [Wikipedia](https://wikipedia.org/) in English, German, French or Esperanto * `YouTube` : YouTube user channel, playlist or search -Plus [many other bridges](bridges/) to enable, thanks to the community +And [many more](bridges/), thanks to the community! Output format === -Output format can take several forms: - -* `Atom` : ATOM Feed, for use in RSS/Feed readers -* `Html` : Simple html page. -* `Json` : Json, for consumption by other applications. -* `Mrss` : MRSS Feed, for use in RSS/Feed readers -* `Plaintext` : raw text (php object, as returned by print_r) - + +RSS-Bridge is capable of producing several output formats: + +* `Atom` : Atom feed, for use in feed readers +* `Html` : Simple HTML page +* `Json` : JSON, for consumption by other applications +* `Mrss` : MRSS feed, for use in feed readers +* `Plaintext` : Raw text, for consumption by other applications + +You can extend RSS-Bridge with your own format, using the [Format API](https://github.com/RSS-Bridge/rss-bridge/wiki/Format-API)! + Screenshot === Welcome screen:  - -RSS-Bridge hashtag (#rss-bridge) search on Twitter, in ATOM format (as displayed by Firefox): + +*** + +RSS-Bridge hashtag (#rss-bridge) search on Twitter, in Atom format (as displayed by Firefox):  - + Requirements === - * PHP 5.6, e.g. `AddHandler application/x-httpd-php56 .php` in `.htaccess` - * `openssl` extension enabled in PHP config (`php.ini`) - * `curl` extension enabled in PHP config (`php.ini`) +RSS-Bridge requires PHP 5.6 or higher with following extensions enabled: + + - [`openssl`](https://secure.php.net/manual/en/book.openssl.php) + - [`libxml`](https://secure.php.net/manual/en/book.libxml.php) + - [`mbstring`](https://secure.php.net/manual/en/book.mbstring.php) + - [`simplexml`](https://secure.php.net/manual/en/book.simplexml.php) + - [`curl`](https://secure.php.net/manual/en/book.curl.php) + - [`json`](https://secure.php.net/manual/en/book.json.php) -Enabling/Disabling bridges +Find more information on our [Wiki](https://github.com/rss-bridge/rss-bridge/wiki) + +Enable / Disable bridges === -By default, the script creates `whitelist.txt` and adds the main bridges (see above). `whitelist.txt` is ignored by git, you can edit it: - * to enable extra bridges (one bridge per line) - * to disable main bridges (remove the line) - * to enable all bridges (just one wildcard `*` as file content) +RSS-Bridge allows you to take full control over which bridges are displayed to the user. That way you can host your own RSS-Bridge service with your favorite collection of bridges! + +Find more information on the [Wiki](https://github.com/RSS-Bridge/rss-bridge/wiki/Whitelisting) -New bridges are disabled by default, so make sure to check regularly what's new and whitelist what you want! +**Notice**: By default RSS-Bridge will only show a small subset of bridges. Make sure to read up on [whitelisting](https://github.com/RSS-Bridge/rss-bridge/wiki/Whitelisting) to unlock the full potential of RSS-Bridge! Deploy === + +Thanks to the community, hosting your own instance of RSS-Bridge is as easy as clicking a button! + [](https://my.scalingo.com/deploy?source=https://github.com/sebsauvage/rss-bridge) - +[](https://cloud.docker.com/stack/deploy/?repo=https://github.com/rss-bridge/rss-bridge) + +Getting involved +=== + +There are many ways for you to getting involved with RSS-Bridge. Here are a few things: + +- Share RSS-Bridge with your friends (Twitter, Facebook, ..._you name it_...) +- Report broken bridges or bugs by opening [Issues](https://github.com/RSS-Bridge/rss-bridge/issues) on GitHub +- Request new features or suggest ideas (via [Issues](https://github.com/RSS-Bridge/rss-bridge/issues)) +- Discuss bugs, features, ideas or [issues](https://github.com/RSS-Bridge/rss-bridge/issues) +- Add new bridges or improve the API +- Improve the [Wiki](https://github.com/RSS-Bridge/rss-bridge/wiki) +- Host an instance of RSS-Bridge for your personal use or make it available to the community :sparkling_heart: + Authors === -We are RSS Bridge Community, a group of developers continuing the project initiated by sebsauvage, webmaster of [sebsauvage.net](http://sebsauvage.net), author of [Shaarli](http://sebsauvage.net/wiki/doku.php?id=php:shaarli) and [ZeroBin](http://sebsauvage.net/wiki/doku.php?id=php:zerobin). - -Patch/contributors : - - * Yves ASTIER ([Draeli](https://github.com/Draeli)) : PHP optimizations, fixes, dynamic brigde/format list with all stuff behind and extend cache system. Mail : contact /at\ yves-astier.com - * [Mitsukarenai](https://github.com/Mitsukarenai) : Initial inspiration, collaborator - * [ArthurHoaro](https://github.com/ArthurHoaro) - * [BoboTiG](https://github.com/BoboTiG) - * [Astalaseven](https://github.com/Astalaseven) - * [qwertygc](https://github.com/qwertygc) - * [Djuuu](https://github.com/Djuuu) - * [Anadrark](https://github.com/Anadrark]) - * [Grummfy](https://github.com/Grummfy) - * [Polopollo](https://github.com/Polopollo) - * [16mhz](https://github.com/16mhz) - * [kranack](https://github.com/kranack) - * [logmanoriginal](https://github.com/logmanoriginal) - * [polo2ro](https://github.com/polo2ro) - * [Riduidel](https://github.com/Riduidel) - * [superbaillot.net](http://superbaillot.net/) - * [vinzv](https://github.com/vinzv) - * [teromene](https://github.com/teromene) - * [nel50n](https://github.com/nel50n) - * [nyutag](https://github.com/nyutag) - * [ORelio](https://github.com/ORelio) - * [Pitchoule](https://github.com/Pitchoule) - * [pit-fgfjiudghdf](https://github.com/pit-fgfjiudghdf) - * [aledeg](https://github.com/aledeg) - * [alexAubin](https://github.com/alexAubin) - * [cnlpete](https://github.com/cnlpete) - * [corenting](https://github.com/corenting) - * [Daiyousei](https://github.com/Daiyousei) - * [erwang](https://github.com/erwang) - * [gsurrel](https://github.com/gsurrel) - * [kraoc](https://github.com/kraoc) - * [lagaisse](https://github.com/lagaisse) - * [az5he6ch](https://github.com/az5he6ch) - * [niawag](https://github.com/niawag) - * [JeremyRand](https://github.com/JeremyRand) - * [mro](https://github.com/mro) + +We are RSS-Bridge community, a group of developers continuing the project initiated by sebsauvage, webmaster of [sebsauvage.net](http://sebsauvage.net), author of [Shaarli](http://sebsauvage.net/wiki/doku.php?id=php:shaarli) and [ZeroBin](http://sebsauvage.net/wiki/doku.php?id=php:zerobin). + +**Contributors** (sorted alphabetically): +<!-- +Use this script to generate the list automatically (using the GitHub API): +https://gist.github.com/LogMANOriginal/da00cd1e5f0ca31cef8e193509b17fd8 +--> + + * [16mhz](https://api.github.com/users/16mhz) + * [Ahiles3005](https://api.github.com/users/Ahiles3005) + * [Albirew](https://api.github.com/users/Albirew) + * [AmauryCarrade](https://api.github.com/users/AmauryCarrade) + * [ArthurHoaro](https://api.github.com/users/ArthurHoaro) + * [Astalaseven](https://api.github.com/users/Astalaseven) + * [Astyan-42](https://api.github.com/users/Astyan-42) + * [Daiyousei](https://api.github.com/users/Daiyousei) + * [Djuuu](https://api.github.com/users/Djuuu) + * [Draeli](https://api.github.com/users/Draeli) + * [EtienneM](https://api.github.com/users/EtienneM) + * [Frenzie](https://api.github.com/users/Frenzie) + * [Ginko-Aloe](https://api.github.com/users/Ginko-Aloe) + * [Glandos](https://api.github.com/users/Glandos) + * [GregThib](https://api.github.com/users/GregThib) + * [Grummfy](https://api.github.com/users/Grummfy) + * [JackNUMBER](https://api.github.com/users/JackNUMBER) + * [JeremyRand](https://api.github.com/users/JeremyRand) + * [Jocker666z](https://api.github.com/users/Jocker666z) + * [LogMANOriginal](https://api.github.com/users/LogMANOriginal) + * [MonsieurPoutounours](https://api.github.com/users/MonsieurPoutounours) + * [ORelio](https://api.github.com/users/ORelio) + * [PaulVayssiere](https://api.github.com/users/PaulVayssiere) + * [Piranhaplant](https://api.github.com/users/Piranhaplant) + * [Riduidel](https://api.github.com/users/Riduidel) + * [Strubbl](https://api.github.com/users/Strubbl) + * [TheRadialActive](https://api.github.com/users/TheRadialActive) + * [TwizzyDizzy](https://api.github.com/users/TwizzyDizzy) + * [WalterBarrett](https://api.github.com/users/WalterBarrett) + * [ZeNairolf](https://api.github.com/users/ZeNairolf) + * [adamchainz](https://api.github.com/users/adamchainz) + * [aledeg](https://api.github.com/users/aledeg) + * [alexAubin](https://api.github.com/users/alexAubin) + * [az5he6ch](https://api.github.com/users/az5he6ch) + * [b1nj](https://api.github.com/users/b1nj) + * [benasse](https://api.github.com/users/benasse) + * [captn3m0](https://api.github.com/users/captn3m0) + * [chemel](https://api.github.com/users/chemel) + * [ckiw](https://api.github.com/users/ckiw) + * [cnlpete](https://api.github.com/users/cnlpete) + * [corenting](https://api.github.com/users/corenting) + * [da2x](https://api.github.com/users/da2x) + * [eMerzh](https://api.github.com/users/eMerzh) + * [em92](https://api.github.com/users/em92) + * [griffaurel](https://api.github.com/users/griffaurel) + * [hunhejj](https://api.github.com/users/hunhejj) + * [j0k3r](https://api.github.com/users/j0k3r) + * [jdigilio](https://api.github.com/users/jdigilio) + * [kranack](https://api.github.com/users/kranack) + * [kraoc](https://api.github.com/users/kraoc) + * [laBecasse](https://api.github.com/users/laBecasse) + * [lagaisse](https://api.github.com/users/lagaisse) + * [lalannev](https://api.github.com/users/lalannev) + * [ldidry](https://api.github.com/users/ldidry) + * [m0zes](https://api.github.com/users/m0zes) + * [matthewseal](https://api.github.com/users/matthewseal) + * [mcbyte-it](https://api.github.com/users/mcbyte-it) + * [mdemoss](https://api.github.com/users/mdemoss) + * [melangue](https://api.github.com/users/melangue) + * [metaMMA](https://api.github.com/users/metaMMA) + * [mickael-bertrand](https://api.github.com/users/mickael-bertrand) + * [mitsukarenai](https://api.github.com/users/mitsukarenai) + * [mro](https://api.github.com/users/mro) + * [mxmehl](https://api.github.com/users/mxmehl) + * [nel50n](https://api.github.com/users/nel50n) + * [niawag](https://api.github.com/users/niawag) + * [pellaeon](https://api.github.com/users/pellaeon) + * [pit-fgfjiudghdf](https://api.github.com/users/pit-fgfjiudghdf) + * [pitchoule](https://api.github.com/users/pitchoule) + * [pmaziere](https://api.github.com/users/pmaziere) + * [prysme01](https://api.github.com/users/prysme01) + * [quentinus95](https://api.github.com/users/quentinus95) + * [qwertygc](https://api.github.com/users/qwertygc) + * [regisenguehard](https://api.github.com/users/regisenguehard) + * [rogerdc](https://api.github.com/users/rogerdc) + * [sebsauvage](https://api.github.com/users/sebsauvage) + * [sublimz](https://api.github.com/users/sublimz) + * [sysadminstory](https://api.github.com/users/sysadminstory) + * [tameroski](https://api.github.com/users/tameroski) + * [teromene](https://api.github.com/users/teromene) + * [triatic](https://api.github.com/users/triatic) + * [wtuuju](https://api.github.com/users/wtuuju) Licenses === -Code is [Public Domain](UNLICENSE). -Including `PHP Simple HTML DOM Parser` under the [MIT License](http://opensource.org/licenses/MIT) +The source code for RSS-Bridge is [Public Domain](UNLICENSE). +RSS-Bridge uses third party libraries with their own license: + + * [`PHP Simple HTML DOM Parser`](http://simplehtmldom.sourceforge.net/) licensed under the [MIT License](http://opensource.org/licenses/MIT) + * [`php-urljoin`](https://github.com/fluffy-critter/php-urljoin) licensed under the [MIT License](http://opensource.org/licenses/MIT) Technical notes === - * There is a cache so that source services won't ban you even if you hammer the rss-bridge with requests. Each bridge can have a different duration for the cache. The `cache` subdirectory will be automatically created and cached objects older than 24 hours get purged. - * To implement a new Bridge, [follow the specifications](https://github.com/RSS-Bridge/rss-bridge/wiki/Bridge-API) and take a look at existing Bridges for examples. - * To enable debug mode (disabling cache and enabling error reporting), create an empty file named `DEBUG` in the root directory (next to `index.php`). - * For more information refer to the [Wiki](https://github.com/RSS-Bridge/rss-bridge/wiki) + + * RSS-Bridge uses caching to prevent services from banning your server for repeatedly updating feeds. The specific cache duration can be different between bridges. Cached files are deleted automatically after 24 hours. + * You can implement your own bridge, [following these instructions](https://github.com/RSS-Bridge/rss-bridge/wiki/Bridge-API). + * You can enable debug mode to disable caching. Find more information on the [Wiki](https://github.com/RSS-Bridge/rss-bridge/wiki/Debug-mode) Rant === @@ -133,10 +217,10 @@ Rant Your catchword is "share", but you don't want us to share. You want to keep us within your walled gardens. That's why you've been removing RSS links from webpages, hiding them deep on your website, or removed feeds entirely, replacing it with crippled or demented proprietary API. **FUCK YOU.** -You're not social when you hamper sharing by removing feeds. You're happy to have customers creating content for your ecosystem, but you don't want this content out - a content you do not even own. Google Takeout is just a gimmick. We want our data to flow, we want RSS or ATOM feeds. +You're not social when you hamper sharing by removing feeds. You're happy to have customers creating content for your ecosystem, but you don't want this content out - a content you do not even own. Google Takeout is just a gimmick. We want our data to flow, we want RSS or Atom feeds. -We want to share with friends, using open protocols: RSS, ATOM, XMPP, whatever. Because no one wants to have *your* service with *your* applications using *your* API force-feeding them. Friends must be free to choose whatever software and service they want. +We want to share with friends, using open protocols: RSS, Atom, XMPP, whatever. Because no one wants to have *your* service with *your* applications using *your* API force-feeding them. Friends must be free to choose whatever software and service they want. We are rebuilding bridges you have wilfully destroyed. -Get your shit together: Put RSS/ATOM back in. +Get your shit together: Put RSS/Atom back in. diff --git a/bridges/ABCTabsBridge.php b/bridges/ABCTabsBridge.php index 2e451e2..ef2c75b 100644 --- a/bridges/ABCTabsBridge.php +++ b/bridges/ABCTabsBridge.php @@ -8,7 +8,7 @@ class ABCTabsBridge extends BridgeAbstract { public function collectData(){ $html = ''; - $html = getSimpleHTMLDOM(static::URI.'tablatures/nouveautes.html') + $html = getSimpleHTMLDOM(static::URI . 'tablatures/nouveautes.html') or returnClientError('No results for this query.'); $table = $html->find('table#myTable', 0)->children(1); diff --git a/bridges/AllocineFRBridge.php b/bridges/AllocineFRBridge.php index 959d0ef..431ae58 100644 --- a/bridges/AllocineFRBridge.php +++ b/bridges/AllocineFRBridge.php @@ -45,7 +45,7 @@ class AllocineFRBridge extends BridgeAbstract { public function getName(){ if(!is_null($this->getInput('category'))) { return self::NAME . ' : ' - .array_search( + . array_search( $this->getInput('category'), self::PARAMETERS[$this->queriedContext]['category']['values'] ); diff --git a/bridges/AmazonBridge.php b/bridges/AmazonBridge.php index cbc6119..b7933e8 100644 --- a/bridges/AmazonBridge.php +++ b/bridges/AmazonBridge.php @@ -52,7 +52,7 @@ class AmazonBridge extends BridgeAbstract { public function getName(){ if(!is_null($this->getInput('tld')) && !is_null($this->getInput('q'))) { - return 'Amazon.'.$this->getInput('tld').': '.$this->getInput('q'); + return 'Amazon.' . $this->getInput('tld') . ': ' . $this->getInput('q'); } return parent::getName(); @@ -60,8 +60,8 @@ class AmazonBridge extends BridgeAbstract { public function collectData() { - $uri = 'https://www.amazon.'.$this->getInput('tld').'/'; - $uri .= 's/?field-keywords='.urlencode($this->getInput('q')).'&sort='.$this->getInput('sort'); + $uri = 'https://www.amazon.' . $this->getInput('tld') . '/'; + $uri .= 's/?field-keywords=' . urlencode($this->getInput('q')) . '&sort=' . $this->getInput('sort'); $html = getSimpleHTMLDOM($uri) or returnServerError('Could not request Amazon.'); @@ -86,7 +86,7 @@ class AmazonBridge extends BridgeAbstract { $price = $element->find('span.s-price', 0); $price = ($price) ? $price->innertext : ''; - $item['content'] = '<img src="'.$image->getAttribute('src').'" /><br />'.$price; + $item['content'] = '<img src="' . $image->getAttribute('src') . '" /><br />' . $price; $this->items[] = $item; } diff --git a/bridges/AmazonPriceTrackerBridge.php b/bridges/AmazonPriceTrackerBridge.php index dd352af..e31a03b 100644 --- a/bridges/AmazonPriceTrackerBridge.php +++ b/bridges/AmazonPriceTrackerBridge.php @@ -92,6 +92,14 @@ class AmazonPriceTrackerBridge extends BridgeAbstract { } } + private function parseDynamicImage($attribute) { + $json = json_decode(html_entity_decode($attribute), true); + + if ($json and count($json) > 0) { + return array_keys($json)[0]; + } + } + /** * Returns a generated image tag for the product */ @@ -99,11 +107,15 @@ class AmazonPriceTrackerBridge extends BridgeAbstract { $imageSrc = $html->find('#main-image-container img', 0); if ($imageSrc) { - $imageSrc = $imageSrc ? $imageSrc->getAttribute('data-old-hires') : ''; - return <<<EOT -<img width="300" style="max-width:300;max-height:300" src="$imageSrc" alt="{$this->title}" /> -EOT; + $hiresImage = $imageSrc->getAttribute('data-old-hires'); + $dynamicImageAttribute = $imageSrc->getAttribute('data-a-dynamic-image'); + $image = $hiresImage ?: $this->parseDynamicImage($dynamicImageAttribute); } + $image = $image ?: 'https://placekitten.com/200/300'; + + return <<<EOT +<img width="300" style="max-width:300;max-height:300" src="$image" alt="{$this->title}" /> +EOT; } /** @@ -116,6 +128,39 @@ EOT; return getSimpleHTMLDOM($uri) ?: returnServerError('Could not request Amazon.'); } + private function scrapePriceFromMetrics($html) { + $asinData = $html->find('#cerberus-data-metrics', 0); + + // <div id="cerberus-data-metrics" style="display: none;" + // data-asin="B00WTHJ5SU" data-asin-price="14.99" data-asin-shipping="0" + // data-asin-currency-code="USD" data-substitute-count="-1" ... /> + if ($asinData) { + return [ + 'price' => $asinData->getAttribute('data-asin-price'), + 'currency' => $asinData->getAttribute('data-asin-currency-code'), + 'shipping' => $asinData->getAttribute('data-asin-shipping') + ]; + } + + return false; + } + + private function scrapePriceGeneric($html) { + $priceDiv = $html->find('span.offer-price', 0) ?: $html->find('.a-color-price', 0); + + preg_match('/^\s*([A-Z]{3}|£|\$)\s?([\d.,]+)\s*$/', $priceDiv->plaintext, $matches); + + if (count($matches) === 3) { + return [ + 'price' => $matches[2], + 'currency' => $matches[1], + 'shipping' => '0' + ]; + } + + return false; + } + /** * Scrape method for Amazon product page * @return [type] [description] @@ -125,23 +170,16 @@ EOT; $this->title = $this->getTitle($html); $imageTag = $this->getImage($html); - $asinData = $html->find('#cerberus-data-metrics', 0); - - // <div id="cerberus-data-metrics" style="display: none;" - // data-asin="B00WTHJ5SU" data-asin-price="14.99" data-asin-shipping="0" - // data-asin-currency-code="USD" data-substitute-count="-1" ... /> - $currency = $asinData->getAttribute('data-asin-currency-code'); - $shipping = $asinData->getAttribute('data-asin-shipping'); - $price = $asinData->getAttribute('data-asin-price'); + $data = $this->scrapePriceFromMetrics($html) ?: $this->scrapePriceGeneric($html); $item = array( 'title' => $this->title, 'uri' => $this->getURI(), - 'content' => "$imageTag<br/>Price: $price $currency", + 'content' => "$imageTag<br/>Price: {$data['price']} {$data['currency']}", ); - if ($shipping !== '0') { - $item['content'] .= "<br>Shipping: $shipping $currency</br>"; + if ($data['shipping'] !== '0') { + $item['content'] .= "<br>Shipping: {$data['shipping']} {$data['currency']}</br>"; } $this->items[] = $item; diff --git a/bridges/AnidexBridge.php b/bridges/AnidexBridge.php new file mode 100644 index 0000000..ae387c9 --- /dev/null +++ b/bridges/AnidexBridge.php @@ -0,0 +1,207 @@ +<?php +class AnidexBridge extends BridgeAbstract { + + const MAINTAINER = 'ORelio'; + const NAME = 'Anidex'; + const URI = 'https://anidex.info/'; + const DESCRIPTION = 'Returns the newest torrents, with optional search criteria.'; + const PARAMETERS = array( + array( + 'id' => array( + 'name' => 'Category', + 'type' => 'list', + 'values' => array( + 'All categories' => '0', + 'Anime' => '1,2,3', + 'Anime - Sub' => '1', + 'Anime - Raw' => '2', + 'Anime - Dub' => '3', + 'Live Action' => '4,5', + 'Live Action - Sub' => '4', + 'Live Action - Raw' => '5', + 'Light Novel' => '6', + 'Manga' => '7,8', + 'Manga - Translated' => '7', + 'Manga - Raw' => '8', + 'Music' => '9,10,11', + 'Music - Lossy' => '9', + 'Music - Lossless' => '10', + 'Music - Video' => '11', + 'Games' => '12', + 'Applications' => '13', + 'Pictures' => '14', + 'Adult Video' => '15', + 'Other' => '16' + ) + ), + 'lang_id' => array( + 'name' => 'Language', + 'type' => 'list', + 'values' => array( + 'All languages' => '0', + 'English' => '1', + 'Japanese' => '2', + 'Polish' => '3', + 'Serbo-Croatian' => '4', + 'Dutch' => '5', + 'Italian' => '6', + 'Russian' => '7', + 'German' => '8', + 'Hungarian' => '9', + 'French' => '10', + 'Finnish' => '11', + 'Vietnamese' => '12', + 'Greek' => '13', + 'Bulgarian' => '14', + 'Spanish (Spain)' => '15', + 'Portuguese (Brazil)' => '16', + 'Portuguese (Portugal)' => '17', + 'Swedish' => '18', + 'Arabic' => '19', + 'Danish' => '20', + 'Chinese (Simplified)' => '21', + 'Bengali' => '22', + 'Romanian' => '23', + 'Czech' => '24', + 'Mongolian' => '25', + 'Turkish' => '26', + 'Indonesian' => '27', + 'Korean' => '28', + 'Spanish (LATAM)' => '29', + 'Persian' => '30', + 'Malaysian' => '31' + ) + ), + 'group_id' => array( + 'name' => 'Group ID', + 'type' => 'number' + ), + 'r' => array( + 'name' => 'Hide Remakes', + 'type' => 'checkbox' + ), + 'b' => array( + 'name' => 'Only Batches', + 'type' => 'checkbox' + ), + 'a' => array( + 'name' => 'Only Authorized', + 'type' => 'checkbox' + ), + 'q' => array( + 'name' => 'Keyword', + 'description' => 'Keyword(s)', + 'type' => 'text' + ), + 'h' => array( + 'name' => 'Adult content', + 'type' => 'list', + 'values' => array( + 'No filter' => '0', + 'Hide +18' => '1', + 'Only +18' => '2' + ) + ) + ) + ); + + public function collectData() { + + // Build Search URL from user-provided parameters + $search_url = self::URI . '?s=upload_timestamp&o=desc'; + foreach (array('id', 'lang_id', 'group_id') as $param_name) { + $param = $this->getInput($param_name); + if (!empty($param) && intval($param) != 0 && ctype_digit(str_replace(',', '', $param))) { + $search_url .= '&' . $param_name . '=' . $param; + } + } + foreach (array('r', 'b', 'a') as $param_name) { + $param = $this->getInput($param_name); + if (!empty($param) && boolval($param)) { + $search_url .= '&' . $param_name . '=1'; + } + } + $query = $this->getInput('q'); + if (!empty($query)) { + $search_url .= '&q=' . urlencode($query); + } + $opt = array(); + $h = $this->getInput('h'); + if (!empty($h) && intval($h) != 0 && ctype_digit($h)) { + $opt[CURLOPT_COOKIE] = 'anidex_h_toggle=' . $h; + } + + // Retrieve torrent listing from search results, which does not contain torrent description + $html = getSimpleHTMLDOM($search_url, array(), $opt) + or returnServerError('Could not request Anidex: ' . $search_url); + $links = $html->find('a'); + $results = array(); + foreach ($links as $link) + if (strpos($link->href, '/torrent/') === 0 && !in_array($link->href, $results)) + $results[] = $link->href; + if (empty($results) && empty($this->getInput('q'))) + returnServerError('No results from Anidex: ' . $search_url); + + //Process each item individually + foreach ($results as $element) { + + //Limit total amount of requests + if(count($this->items) >= 20) { + break; + } + + $torrent_id = str_replace('/torrent/', '', $element); + + //Ignore entries without valid torrent ID + if ($torrent_id != 0 && ctype_digit($torrent_id)) { + + //Retrieve data for this torrent ID + $item_uri = self::URI . 'torrent/' . $torrent_id; + + //Retrieve full description from torrent page + if ($item_html = getSimpleHTMLDOMCached($item_uri)) { + + //Retrieve data from page contents + $item_title = str_replace(' (Torrent) - AniDex ', '', $item_html->find('title', 0)->plaintext); + $item_desc = $item_html->find('div.panel-body', 0); + $item_author = trim($item_html->find('span.fa-user', 0)->parent()->plaintext); + $item_date = strtotime(trim($item_html->find('span.fa-clock', 0)->parent()->plaintext)); + $item_image = $this->getURI() . 'images/user_logos/default.png'; + + //Check for description-less torrent andn optionally extract image + $desc_title_found = false; + foreach ($item_html->find('h3.panel-title') as $h3) { + if (strpos($h3, 'Description') !== false) { + $desc_title_found = true; + break; + } + } + if ($desc_title_found) { + //Retrieve image for thumbnail or generic logo fallback + foreach ($item_desc->find('img') as $img) { + if (strpos($img->src, 'prez') === false) { + $item_image = $img->src; + break; + } + } + $item_desc = trim($item_desc->innertext); + } else { + $item_desc = '<em>No description.</em>'; + } + + //Build and add final item + $item = array(); + $item['uri'] = $item_uri; + $item['title'] = $item_title; + $item['author'] = $item_author; + $item['timestamp'] = $item_date; + $item['enclosures'] = array($item_image); + $item['content'] = $item_desc; + $this->items[] = $item; + } + } + $element = null; + } + $results = null; + } +} diff --git a/bridges/AnimeUltimeBridge.php b/bridges/AnimeUltimeBridge.php index 6c5427e..5c719b9 100644 --- a/bridges/AnimeUltimeBridge.php +++ b/bridges/AnimeUltimeBridge.php @@ -5,7 +5,7 @@ class AnimeUltimeBridge extends BridgeAbstract { const NAME = 'Anime-Ultime'; const URI = 'http://www.anime-ultime.net/'; const CACHE_TIMEOUT = 10800; // 3h - const DESCRIPTION = 'Returns the 10 newest releases posted on Anime-Ultime'; + const DESCRIPTION = 'Returns the newest releases posted on Anime-Ultime.'; const PARAMETERS = array( array( 'type' => array( 'name' => 'Type', @@ -65,6 +65,13 @@ class AnimeUltimeBridge extends BridgeAbstract { $item_link_element = $release->find('td', 0)->find('a', 0); $item_uri = self::URI . $item_link_element->href; $item_name = html_entity_decode($item_link_element->plaintext); + + $item_image = self::URI . substr( + $item_link_element->onmouseover, + 37, + strpos($item_link_element->onmouseover, ' ', 37) - 37 + ); + $item_episode = html_entity_decode( str_pad( $release->find('td', 1)->plaintext, @@ -79,8 +86,7 @@ class AnimeUltimeBridge extends BridgeAbstract { if(!empty($item_uri)) { - // Retrieve description from description page and - // convert relative image src info absolute image src + // Retrieve description from description page $html_item = getContents($item_uri) or returnServerError('Could not request Anime-Ultime: ' . $item_uri); $item_description = substr( @@ -91,10 +97,9 @@ class AnimeUltimeBridge extends BridgeAbstract { 0, strpos($item_description, '<div id="table">') ); - $item_description = str_replace( - 'src="images', 'src="' . self::URI . 'images', - $item_description - ); + + // Convert relative image src into absolute image src, remove line breaks + $item_description = defaultLinkTo($item_description, self::URI); $item_description = str_replace("\r", '', $item_description); $item_description = str_replace("\n", '', $item_description); $item_description = utf8_encode($item_description); @@ -105,6 +110,7 @@ class AnimeUltimeBridge extends BridgeAbstract { $item['title'] = $item_name . ' ' . $item_type . ' ' . $item_episode; $item['author'] = $item_fansub; $item['timestamp'] = $item_date; + $item['enclosures'] = array($item_image); $item['content'] = $item_description; $this->items[] = $item; $processedOK++; diff --git a/bridges/Arte7Bridge.php b/bridges/Arte7Bridge.php index 16952dc..12588c6 100644 --- a/bridges/Arte7Bridge.php +++ b/bridges/Arte7Bridge.php @@ -28,6 +28,13 @@ class Arte7Bridge extends BridgeAbstract { ) ) ), + 'Collection (Français)' => array( + 'colfr' => array( + 'name' => 'Collection id', + 'required' => true, + 'title' => 'ex. RC-014095 pour https://www.arte.tv/fr/videos/RC-014095/blow-up/' + ) + ), 'Catégorie (Allemand)' => array( 'catde' => array( 'type' => 'list', @@ -45,6 +52,13 @@ class Arte7Bridge extends BridgeAbstract { 'Sonstiges' => 'AUT' ) ) + ), + 'Collection (Allemand)' => array( + 'colde' => array( + 'name' => 'Collection id', + 'required' => true, + 'title' => 'ex. RC-014095 pour https://www.arte.tv/de/videos/RC-014095/blow-up/' + ) ) ); @@ -54,15 +68,24 @@ class Arte7Bridge extends BridgeAbstract { $category = $this->getInput('catfr'); $lang = 'fr'; break; + case 'Collection (Français)': + $lang = 'fr'; + $collectionId = $this->getInput('colfr'); + break; case 'Catégorie (Allemand)': $category = $this->getInput('catde'); $lang = 'de'; break; + case 'Collection (Allemand)': + $lang = 'de'; + $collectionId = $this->getInput('colde'); + break; } $url = 'https://api.arte.tv/api/opa/v3/videos?sort=-lastModified&limit=10&language=' . $lang - . ($category != null ? '&category.code=' . $category : ''); + . ($category != null ? '&category.code=' . $category : '') + . ($collectionId != null ? '&collections.collectionId=' . $collectionId : ''); $header = array( 'Authorization: Bearer ' . self::API_TOKEN diff --git a/bridges/AskfmBridge.php b/bridges/AskfmBridge.php index e227461..b76d51b 100644 --- a/bridges/AskfmBridge.php +++ b/bridges/AskfmBridge.php @@ -1,7 +1,7 @@ <?php class AskfmBridge extends BridgeAbstract { - const MAINTAINER = 'az5he6ch'; + const MAINTAINER = 'az5he6ch, logmanoriginal'; const NAME = 'Ask.fm Answers'; const URI = 'https://ask.fm/'; const CACHE_TIMEOUT = 300; //5 min @@ -19,39 +19,39 @@ class AskfmBridge extends BridgeAbstract { $html = getSimpleHTMLDOM($this->getURI()) or returnServerError('Requested username can\'t be found.'); - foreach($html->find('div.streamItem-answer') as $element) { + $html = defaultLinkTo($html, self::URI); + + foreach($html->find('article.streamItem-answer') as $element) { $item = array(); - $item['uri'] = self::URI . $element->find('a.streamItemsAge', 0)->href; - $question = trim($element->find('h1.streamItemContent-question', 0)->innertext); + $item['uri'] = $element->find('a.streamItem_meta', 0)->href; + $question = trim($element->find('header.streamItem_header', 0)->innertext); $item['title'] = trim( - htmlspecialchars_decode($element->find('h1.streamItemContent-question', 0)->plaintext, + htmlspecialchars_decode($element->find('header.streamItem_header', 0)->plaintext, ENT_QUOTES ) ); - $answer = trim($element->find('p.streamItemContent-answer', 0)->innertext); + $item['timestamp'] = strtotime($element->find('time', 0)->datetime); - // Doesn't work, DOM parser doesn't seem to like data-hint, dunno why - #$item['update'] = $element->find('a.streamitemsage',0)->data-hint; + $answer = trim($element->find('div.streamItem_content', 0)->innertext); // This probably should be cleaned up, especially for YouTube embeds - $visual = $element->find('div.streamItemContent-visual', 0)->innertext; - //Fix tracking links, also doesn't work + if($visual = $element->find('div.streamItem_visual', 0)) { + $visual = $visual->innertext; + } + + // Fix tracking links, also doesn't work foreach($element->find('a') as $link) { if(strpos($link->href, 'l.ask.fm') !== false) { - - // Too slow - #$link->href = str_replace('#_=_', '', get_headers($link->href, 1)['Location']); - $link->href = $link->plaintext; } } - $content = '<p>' . $question . '</p><p>' . $answer . '</p><p>' . $visual . '</p>'; - // Fix relative links without breaking // scheme used by YouTube stuff - $content = preg_replace('#href="\/(?!\/)#', 'href="' . self::URI, $content); - $item['content'] = $content; + $item['content'] = '<p>' . $question + . '</p><p>' . $answer + . '</p><p>' . $visual . '</p>'; + $this->items[] = $item; } } @@ -66,7 +66,7 @@ class AskfmBridge extends BridgeAbstract { public function getURI(){ if(!is_null($this->getInput('u'))) { - return self::URI . urlencode($this->getInput('u')) . '/answers/more?page=0'; + return self::URI . urlencode($this->getInput('u')); } return parent::getURI(); diff --git a/bridges/AutoJMBridge.php b/bridges/AutoJMBridge.php new file mode 100644 index 0000000..598f043 --- /dev/null +++ b/bridges/AutoJMBridge.php @@ -0,0 +1,65 @@ +<?php + +class AutoJMBridge extends BridgeAbstract { + + const NAME = 'AutoJM'; + const URI = 'http://www.autojm.fr/'; + const DESCRIPTION = 'Suivre les offres de véhicules proposés par AutoJM en fonction des critères de filtrages'; + const MAINTAINER = 'sysadminstory'; + const PARAMETERS = array( + 'Afficher les offres de véhicules disponible en fonction des critères du site AutoJM' => array( + 'url' => array( + 'name' => 'URL de la recherche', + 'type' => 'text', + 'required' => true, + 'title' => 'URL d\'une recherche avec filtre de véhicules sans le http://www.autojm.fr/', + 'exampleValue' => 'gammes/index/398?order_by=finition_asc&energie[]=3&transmission[]=2&dispo=all' + ) + ) + ); + const CACHE_TIMEOUT = 3600; + + public function getIcon() { + return self::URI . 'assets/images/favicon.ico'; + } + + public function collectData() { + $html = getSimpleHTMLDOM(self::URI . $this->getInput('url')) + or returnServerError('Could not request AutoJM.'); + $list = $html->find('div[class*=ligne_modele]'); + foreach($list as $element) { + $image = $element->find('img[class=width-100]', 0)->src; + $serie = $element->find('div[class=serie]', 0)->find('span', 0)->plaintext; + $url = $element->find('div[class=serie]', 0)->find('a[class=btn_ligne color-black]', 0)->href; + if($element->find('div[class*=hasStock-info]', 0) != null) { + $dispo = 'Disponible'; + } else { + $dispo = 'Sur commande'; + } + $carburant = str_replace('dispo |', '', $element->find('div[class=carburant]', 0)->plaintext); + $transmission = $element->find('div[class*=bv]', 0)->plaintext; + $places = $element->find('div[class*=places]', 0)->plaintext; + $portes = $element->find('div[class*=nb_portes]', 0)->plaintext; + $carosserie = $element->find('div[class*=coloris]', 0)->plaintext; + $remise = $element->find('div[class*=remise]', 0)->plaintext; + $prix = $element->find('div[class*=prixjm]', 0)->plaintext; + + $item = array(); + $item['uri'] = $url; + $item['title'] = $serie; + $item['content'] = '<p><img style="vertical-align:middle ; padding: 10px" src="' . $image . '" />' . $serie . '</p>'; + $item['content'] .= '<ul><li>Disponibilité : ' . $dispo . '</li>'; + $item['content'] .= '<li>Carburant : ' . $carburant . '</li>'; + $item['content'] .= '<li>Transmission : ' . $transmission . '</li>'; + $item['content'] .= '<li>Nombre de places : ' . $places . '</li>'; + $item['content'] .= '<li>Nombre de portes : ' . $portes . '</li>'; + $item['content'] .= '<li>Série : ' . $serie . '</li>'; + $item['content'] .= '<li>Carosserie : ' . $carosserie . '</li>'; + $item['content'] .= '<li>Remise : ' . $remise . '</li>'; + $item['content'] .= '<li>Prix : ' . $prix . '</li></ul>'; + + $this->items[] = $item; + } + + } +} diff --git a/bridges/BAEBridge.php b/bridges/BAEBridge.php new file mode 100644 index 0000000..caa2cf7 --- /dev/null +++ b/bridges/BAEBridge.php @@ -0,0 +1,265 @@ +<?php +class BAEBridge extends BridgeAbstract { + const MAINTAINER = 'couraudt'; + const NAME = 'Bourse Aux Equipiers Bridge'; + const URI = 'https://www.bourse-aux-equipiers.com'; + const DESCRIPTION = 'Returns the newest sailing offers.'; + const PARAMETERS = array( + array( + 'keyword' => array( + 'name' => 'Filtrer par mots clés', + 'title' => 'Entrez le mot clé à filtrer ici' + ), + 'type' => array( + 'name' => 'Type de recherche', + 'title' => 'Afficher seuleument un certain type d\'annonce', + 'type' => 'list', + 'values' => array( + 'Toutes les annonces' => false, + 'Les embarquements' => 'boat', + 'Les skippers' => 'skipper', + 'Les équipiers' => 'crew' + ) + ) + ) + ); + + public function collectData() { + $url = $this->getURI(); + $html = getSimpleHTMLDOM($url) or returnClientError('No results for this query.'); + + $annonces = $html->find('main article'); + foreach ($annonces as $annonce) { + $detail = $annonce->find('footer a', 0); + + $htmlDetail = getSimpleHTMLDOMCached(parent::getURI() . $detail->href); + if (!$htmlDetail) + continue; + + $item = array(); + + $item['title'] = $annonce->find('header h2', 0)->plaintext; + $item['uri'] = parent::getURI() . $detail->href; + + $content = $htmlDetail->find('article p', 0)->innertext; + if (!empty($this->getInput('keyword'))) { + $keyword = $this->remove_accents(strtolower($this->getInput('keyword'))); + $cleanTitle = $this->remove_accents(strtolower($item['title'])); + if (strpos($cleanTitle, $keyword) === false) { + $cleanContent = $this->remove_accents(strtolower($content)); + if (strpos($cleanContent, $keyword) === false) { + continue; + } + } + } + + $content .= '<hr>'; + $content .= $htmlDetail->find('section', 0)->innertext; + $content = str_replace('src="/', 'src="' . parent::getURI() . '/', $content); + $content = str_replace('href="/', 'href="' . parent::getURI() . '/', $content); + $item['content'] = $content; + $image = $htmlDetail->find('#zoom', 0); + if ($image) { + $item['enclosures'] = array(parent::getURI() . $image->getAttribute('src')); + } + $this->items[] = $item; + } + } + + public function getURI() { + $uri = parent::getURI(); + if (!empty($this->getInput('type'))) { + if ($this->getInput('type') == 'boat') { + $uri .= '/embarquements.html'; + } elseif ($this->getInput('type') == 'skipper') { + $uri .= '/skippers.html'; + } else { + $uri .= '/equipiers.html'; + } + } + + return $uri; + } + + private function remove_accents($string) { + $chars = array( + // Decompositions for Latin-1 Supplement + 'ª' => 'a', 'º' => 'o', + 'À' => 'A', 'Á' => 'A', + 'Â' => 'A', 'Ã' => 'A', + 'Ä' => 'A', 'Å' => 'A', + 'Æ' => 'AE', 'Ç' => 'C', + 'È' => 'E', 'É' => 'E', + 'Ê' => 'E', 'Ë' => 'E', + 'Ì' => 'I', 'Í' => 'I', + 'Î' => 'I', 'Ï' => 'I', + 'Ð' => 'D', 'Ñ' => 'N', + 'Ò' => 'O', 'Ó' => 'O', + 'Ô' => 'O', 'Õ' => 'O', + 'Ö' => 'O', 'Ù' => 'U', + 'Ú' => 'U', 'Û' => 'U', + 'Ü' => 'U', 'Ý' => 'Y', + 'Þ' => 'TH', 'ß' => 's', + 'à' => 'a', 'á' => 'a', + 'â' => 'a', 'ã' => 'a', + 'ä' => 'a', 'å' => 'a', + 'æ' => 'ae', 'ç' => 'c', + 'è' => 'e', 'é' => 'e', + 'ê' => 'e', 'ë' => 'e', + 'ì' => 'i', 'í' => 'i', + 'î' => 'i', 'ï' => 'i', + 'ð' => 'd', 'ñ' => 'n', + 'ò' => 'o', 'ó' => 'o', + 'ô' => 'o', 'õ' => 'o', + 'ö' => 'o', 'ø' => 'o', + 'ù' => 'u', 'ú' => 'u', + 'û' => 'u', 'ü' => 'u', + 'ý' => 'y', 'þ' => 'th', + 'ÿ' => 'y', 'Ø' => 'O', + // Decompositions for Latin Extended-A + 'Ā' => 'A', 'ā' => 'a', + 'Ă' => 'A', 'ă' => 'a', + 'Ą' => 'A', 'ą' => 'a', + 'Ć' => 'C', 'ć' => 'c', + 'Ĉ' => 'C', 'ĉ' => 'c', + 'Ċ' => 'C', 'ċ' => 'c', + 'Č' => 'C', 'č' => 'c', + 'Ď' => 'D', 'ď' => 'd', + 'Đ' => 'D', 'đ' => 'd', + 'Ē' => 'E', 'ē' => 'e', + 'Ĕ' => 'E', 'ĕ' => 'e', + 'Ė' => 'E', 'ė' => 'e', + 'Ę' => 'E', 'ę' => 'e', + 'Ě' => 'E', 'ě' => 'e', + 'Ĝ' => 'G', 'ĝ' => 'g', + 'Ğ' => 'G', 'ğ' => 'g', + 'Ġ' => 'G', 'ġ' => 'g', + 'Ģ' => 'G', 'ģ' => 'g', + 'Ĥ' => 'H', 'ĥ' => 'h', + 'Ħ' => 'H', 'ħ' => 'h', + 'Ĩ' => 'I', 'ĩ' => 'i', + 'Ī' => 'I', 'ī' => 'i', + 'Ĭ' => 'I', 'ĭ' => 'i', + 'Į' => 'I', 'į' => 'i', + 'İ' => 'I', 'ı' => 'i', + 'IJ' => 'IJ', 'ij' => 'ij', + 'Ĵ' => 'J', 'ĵ' => 'j', + 'Ķ' => 'K', 'ķ' => 'k', + 'ĸ' => 'k', 'Ĺ' => 'L', + 'ĺ' => 'l', 'Ļ' => 'L', + 'ļ' => 'l', 'Ľ' => 'L', + 'ľ' => 'l', 'Ŀ' => 'L', + 'ŀ' => 'l', 'Ł' => 'L', + 'ł' => 'l', 'Ń' => 'N', + 'ń' => 'n', 'Ņ' => 'N', + 'ņ' => 'n', 'Ň' => 'N', + 'ň' => 'n', 'ʼn' => 'n', + 'Ŋ' => 'N', 'ŋ' => 'n', + 'Ō' => 'O', 'ō' => 'o', + 'Ŏ' => 'O', 'ŏ' => 'o', + 'Ő' => 'O', 'ő' => 'o', + 'Œ' => 'OE', 'œ' => 'oe', + 'Ŕ' => 'R', 'ŕ' => 'r', + 'Ŗ' => 'R', 'ŗ' => 'r', + 'Ř' => 'R', 'ř' => 'r', + 'Ś' => 'S', 'ś' => 's', + 'Ŝ' => 'S', 'ŝ' => 's', + 'Ş' => 'S', 'ş' => 's', + 'Š' => 'S', 'š' => 's', + 'Ţ' => 'T', 'ţ' => 't', + 'Ť' => 'T', 'ť' => 't', + 'Ŧ' => 'T', 'ŧ' => 't', + 'Ũ' => 'U', 'ũ' => 'u', + 'Ū' => 'U', 'ū' => 'u', + 'Ŭ' => 'U', 'ŭ' => 'u', + 'Ů' => 'U', 'ů' => 'u', + 'Ű' => 'U', 'ű' => 'u', + 'Ų' => 'U', 'ų' => 'u', + 'Ŵ' => 'W', 'ŵ' => 'w', + 'Ŷ' => 'Y', 'ŷ' => 'y', + 'Ÿ' => 'Y', 'Ź' => 'Z', + 'ź' => 'z', 'Ż' => 'Z', + 'ż' => 'z', 'Ž' => 'Z', + 'ž' => 'z', 'ſ' => 's', + // Decompositions for Latin Extended-B + 'Ș' => 'S', 'ș' => 's', + 'Ț' => 'T', 'ț' => 't', + // Euro Sign + '€' => 'E', + // GBP (Pound) Sign + '£' => '', + // Vowels with diacritic (Vietnamese) + // unmarked + 'Ơ' => 'O', 'ơ' => 'o', + 'Ư' => 'U', 'ư' => 'u', + // grave accent + 'Ầ' => 'A', 'ầ' => 'a', + 'Ằ' => 'A', 'ằ' => 'a', + 'Ề' => 'E', 'ề' => 'e', + 'Ồ' => 'O', 'ồ' => 'o', + 'Ờ' => 'O', 'ờ' => 'o', + 'Ừ' => 'U', 'ừ' => 'u', + 'Ỳ' => 'Y', 'ỳ' => 'y', + // hook + 'Ả' => 'A', 'ả' => 'a', + 'Ẩ' => 'A', 'ẩ' => 'a', + 'Ẳ' => 'A', 'ẳ' => 'a', + 'Ẻ' => 'E', 'ẻ' => 'e', + 'Ể' => 'E', 'ể' => 'e', + 'Ỉ' => 'I', 'ỉ' => 'i', + 'Ỏ' => 'O', 'ỏ' => 'o', + 'Ổ' => 'O', 'ổ' => 'o', + 'Ở' => 'O', 'ở' => 'o', + 'Ủ' => 'U', 'ủ' => 'u', + 'Ử' => 'U', 'ử' => 'u', + 'Ỷ' => 'Y', 'ỷ' => 'y', + // tilde + 'Ẫ' => 'A', 'ẫ' => 'a', + 'Ẵ' => 'A', 'ẵ' => 'a', + 'Ẽ' => 'E', 'ẽ' => 'e', + 'Ễ' => 'E', 'ễ' => 'e', + 'Ỗ' => 'O', 'ỗ' => 'o', + 'Ỡ' => 'O', 'ỡ' => 'o', + 'Ữ' => 'U', 'ữ' => 'u', + 'Ỹ' => 'Y', 'ỹ' => 'y', + // acute accent + 'Ấ' => 'A', 'ấ' => 'a', + 'Ắ' => 'A', 'ắ' => 'a', + 'Ế' => 'E', 'ế' => 'e', + 'Ố' => 'O', 'ố' => 'o', + 'Ớ' => 'O', 'ớ' => 'o', + 'Ứ' => 'U', 'ứ' => 'u', + // dot below + 'Ạ' => 'A', 'ạ' => 'a', + 'Ậ' => 'A', 'ậ' => 'a', + 'Ặ' => 'A', 'ặ' => 'a', + 'Ẹ' => 'E', 'ẹ' => 'e', + 'Ệ' => 'E', 'ệ' => 'e', + 'Ị' => 'I', 'ị' => 'i', + 'Ọ' => 'O', 'ọ' => 'o', + 'Ộ' => 'O', 'ộ' => 'o', + 'Ợ' => 'O', 'ợ' => 'o', + 'Ụ' => 'U', 'ụ' => 'u', + 'Ự' => 'U', 'ự' => 'u', + 'Ỵ' => 'Y', 'ỵ' => 'y', + // Vowels with diacritic (Chinese, Hanyu Pinyin) + 'ɑ' => 'a', + // macron + 'Ǖ' => 'U', 'ǖ' => 'u', + // acute accent + 'Ǘ' => 'U', 'ǘ' => 'u', + // caron + 'Ǎ' => 'A', 'ǎ' => 'a', + 'Ǐ' => 'I', 'ǐ' => 'i', + 'Ǒ' => 'O', 'ǒ' => 'o', + 'Ǔ' => 'U', 'ǔ' => 'u', + 'Ǚ' => 'U', 'ǚ' => 'u', + // grave accent + 'Ǜ' => 'U', 'ǜ' => 'u', + ); + + $string = strtr($string, $chars); + + return $string; + } +} diff --git a/bridges/BandcampBridge.php b/bridges/BandcampBridge.php index 0527da0..9c8d436 100644 --- a/bridges/BandcampBridge.php +++ b/bridges/BandcampBridge.php @@ -14,6 +14,10 @@ class BandcampBridge extends BridgeAbstract { ) )); + public function getIcon() { + return 'https://s4.bcbits.com/img/bc_favicon.ico'; + } + public function collectData(){ $html = getSimpleHTMLDOM($this->getURI()) or returnServerError('No results for this query.'); diff --git a/bridges/BlaguesDeMerdeBridge.php b/bridges/BlaguesDeMerdeBridge.php index 3ae59a1..e4af8d5 100644 --- a/bridges/BlaguesDeMerdeBridge.php +++ b/bridges/BlaguesDeMerdeBridge.php @@ -1,31 +1,47 @@ <?php class BlaguesDeMerdeBridge extends BridgeAbstract { - const MAINTAINER = 'superbaillot.net'; + const MAINTAINER = 'superbaillot.net, logmanoriginal'; const NAME = 'Blagues De Merde'; const URI = 'http://www.blaguesdemerde.fr/'; const CACHE_TIMEOUT = 7200; // 2h const DESCRIPTION = 'Blagues De Merde'; + public function getIcon() { + return self::URI . 'assets/img/favicon.ico'; + } + public function collectData(){ + $html = getSimpleHTMLDOM(self::URI) or returnServerError('Could not request BDM.'); - foreach($html->find('article.joke_contener') as $element) { + foreach($html->find('div.blague') as $element) { + $item = array(); - $temp = $element->find('a'); - - if(isset($temp[2])) { - $item['content'] = trim($element->find('div.joke_text_contener', 0)->innertext); - $uri = $temp[2]->href; - $item['uri'] = $uri; - $item['title'] = substr($uri, (strrpos($uri, '/') + 1)); - $date = $element->find('li.bdm_date', 0)->innertext; - $time = mktime(0, 0, 0, substr($date, 3, 2), substr($date, 0, 2), substr($date, 6, 4)); - $item['timestamp'] = $time; - $item['author'] = $element->find('li.bdm_pseudo', 0)->innertext; - $this->items[] = $item; - } + + $item['uri'] = static::URI . '#' . $element->id; + $item['author'] = $element->find('div[class="blague-footer"] p strong', 0)->plaintext; + + // Let the title be everything up to the first <br> + $item['title'] = trim(explode("\n", $element->find('div.text', 0)->plaintext)[0]); + + $item['content'] = strip_tags($element->find('div.text', 0)); + + // timestamp is part of: + // <p>Par <strong>{author}</strong> le {date} dans <strong>{category}</strong></p> + preg_match( + '/.+le(.+)dans.*/', + $element->find('div[class="blague-footer"]', 0)->plaintext, + $matches + ); + + $item['timestamp'] = strtotime($matches[1]); + + $this->items[] = $item; + } + } + } diff --git a/bridges/BloombergBridge.php b/bridges/BloombergBridge.php index 8aff0ec..9eb1219 100644 --- a/bridges/BloombergBridge.php +++ b/bridges/BloombergBridge.php @@ -31,6 +31,10 @@ class BloombergBridge extends BridgeAbstract return parent::getName(); } + public function getIcon() { + return 'https://assets.bwbx.io/s3/javelin/public/hub/images/favicon-black-63fe5249d3.png'; + } + public function collectData() { switch($this->queriedContext) { diff --git a/bridges/BundesbankBridge.php b/bridges/BundesbankBridge.php new file mode 100644 index 0000000..59c07e0 --- /dev/null +++ b/bridges/BundesbankBridge.php @@ -0,0 +1,87 @@ +<?php +class BundesbankBridge extends BridgeAbstract { + + const PARAM_LANG = 'lang'; + + const LANG_EN = 'en'; + const LANG_DE = 'de'; + + const NAME = 'Bundesbank Bridge'; + const URI = 'https://www.bundesbank.de/'; + const DESCRIPTION = 'Returns the latest studies of the Bundesbank (Germany)'; + const MAINTAINER = 'logmanoriginal'; + const CACHE_TIMEOUT = 86400; // 24 hours + + const PARAMETERS = array( + array( + self::PARAM_LANG => array( + 'name' => 'Language', + 'type' => 'list', + 'required' => true, + 'defaultValue' => self::LANG_DE, + 'values' => array( + 'English' => self::LANG_EN, + 'Deutsch' => self::LANG_DE + ) + ) + ) + ); + + public function getIcon() { + return self::URI . 'resource/crblob/1890/a7f48ee0ae35348748121770ba3ca009/mL/favicon-ico-data.ico'; + } + + public function getURI() { + switch($this->getInput(self::PARAM_LANG)) { + case self::LANG_EN: return self::URI . 'en/publications/reports/studies'; + case self::LANG_DE: return self::URI . 'de/publikationen/berichte/studien'; + } + + return parent::getURI(); + } + + public function collectData() { + + $html = getSimpleHTMLDOM($this->getURI()) + or returnServerError('No response for ' . $this->getURI()); + + $html = defaultLinkTo($html, $this->getURI()); + + foreach($html->find('ul.resultlist li') as $study) { + $item = array(); + + $item['uri'] = $study->find('.teasable__link', 0)->href; + + // Get title without child elements (i.e. subtitle) + $title = $study->find('.teasable__title div.h2', 0); + + foreach($title->children as &$child) { + $child->outertext = ''; + } + + $item['title'] = $title->innertext; + + // Add subtitle to the content if it exists + $item['content'] = ''; + + if($subtitle = $study->find('.teasable__subtitle', 0)) { + $item['content'] .= '<strong>' . $study->find('.teasable__subtitle', 0)->plaintext . '</strong>'; + } + + $item['content'] .= '<p>' . $study->find('.teasable__text', 0)->plaintext . '</p>'; + + $item['timestamp'] = strtotime($study->find('.teasable__date', 0)->plaintext); + + // Downloads and older studies don't have images + if($study->find('.teasable__image', 0)) { + $item['enclosures'] = array( + $study->find('.teasable__image img', 0)->src + ); + } + + $this->items[] = $item; + } + + } + +} diff --git a/bridges/CADBridge.php b/bridges/CADBridge.php deleted file mode 100644 index e88cbbb..0000000 --- a/bridges/CADBridge.php +++ /dev/null @@ -1,45 +0,0 @@ -<?php -class CADBridge extends FeedExpander { - const MAINTAINER = 'nyutag'; - const NAME = 'CAD Bridge'; - const URI = 'http://www.cad-comic.com/'; - const CACHE_TIMEOUT = 7200; //2h - const DESCRIPTION = 'Returns the newest articles.'; - - public function collectData(){ - $this->collectExpandableDatas('http://cdn2.cad-comic.com/rss.xml', 10); - } - - protected function parseItem($newsItem){ - $item = parent::parseItem($newsItem); - $item['content'] = $this->extractCADContent($item['uri']); - return $item; - } - - private function extractCADContent($url) { - $html3 = getSimpleHTMLDOMCached($url); - - // The request might fail due to missing https support or wrong URL - if($html3 == false) - return 'Daily comic not released yet'; - - $htmlpart = explode('/', $url); - - switch ($htmlpart[3]) { - case 'cad': - preg_match_all('/http:\/\/cdn2\.cad-comic\.com\/comics\/cad-\S*png/', $html3, $url2); - break; - case 'sillies': - preg_match_all('/http:\/\/cdn2\.cad-comic\.com\/comics\/sillies-\S*gif/', $html3, $url2); - break; - default: - return 'Daily comic not released yet'; - } - $img = implode($url2[0]); - $html3->clear(); - unset($html3); - if ($img == '') - return 'Daily comic not released yet'; - return '<img src="' . $img . '"/>'; - } -} diff --git a/bridges/CNETBridge.php b/bridges/CNETBridge.php index eefb705..564b817 100644 --- a/bridges/CNETBridge.php +++ b/bridges/CNETBridge.php @@ -3,91 +3,107 @@ class CNETBridge extends BridgeAbstract { const MAINTAINER = 'ORelio'; const NAME = 'CNET News'; - const URI = 'http://www.cnet.com/'; - const CACHE_TIMEOUT = 1800; // 30min - const DESCRIPTION = 'Returns the newest articles. <br /> You may specify a -topic found in some section URLs, else all topics are selected.'; - - const PARAMETERS = array( array( - 'topic' => array( - 'name' => 'Topic name' + const URI = 'https://www.cnet.com/'; + const CACHE_TIMEOUT = 3600; // 1h + const DESCRIPTION = 'Returns the newest articles.'; + const PARAMETERS = array( + array( + 'topic' => array( + 'name' => 'Topic', + 'type' => 'list', + 'values' => array( + 'All articles' => '', + 'Apple' => 'apple', + 'Google' => 'google', + 'Microsoft' => 'tags-microsoft', + 'Computers' => 'topics-computers', + 'Mobile' => 'topics-mobile', + 'Sci-Tech' => 'topics-sci-tech', + 'Security' => 'topics-security', + 'Internet' => 'topics-internet', + 'Tech Industry' => 'topics-tech-industry' + ) + ) ) - )); + ); - public function collectData(){ + private function cleanArticle($article_html) { + $offset_p = strpos($article_html, '<p>'); + $offset_figure = strpos($article_html, '<figure'); + $offset = ($offset_figure < $offset_p ? $offset_figure : $offset_p); + $article_html = substr($article_html, $offset); + $article_html = str_replace('href="/', 'href="' . self::URI, $article_html); + $article_html = str_replace(' height="0"', '', $article_html); + $article_html = str_replace('<noscript>', '', $article_html); + $article_html = str_replace('</noscript>', '', $article_html); + $article_html = StripWithDelimiters($article_html, '<a class="clickToEnlarge', '</a>'); + $article_html = stripWithDelimiters($article_html, '<span class="nowPlaying', '</span>'); + $article_html = stripWithDelimiters($article_html, '<span class="duration', '</span>'); + $article_html = stripWithDelimiters($article_html, '<script', '</script>'); + $article_html = stripWithDelimiters($article_html, '<svg', '</svg>'); + return $article_html; + } - function extractFromDelimiters($string, $start, $end){ - if(strpos($string, $start) !== false) { - $section_retrieved = substr($string, strpos($string, $start) + strlen($start)); - $section_retrieved = substr($section_retrieved, 0, strpos($section_retrieved, $end)); - return $section_retrieved; - } + public function collectData() { - return false; - } + // Retrieve and check user input + $topic = str_replace('-', '/', $this->getInput('topic')); + if (!empty($topic) && (substr_count($topic, '/') > 1 || !ctype_alpha(str_replace('/', '', $topic)))) + returnClientError('Invalid topic: ' . $topic); - function stripWithDelimiters($string, $start, $end){ - while(strpos($string, $start) !== false) { - $section_to_remove = substr($string, strpos($string, $start)); - $section_to_remove = substr($section_to_remove, 0, strpos($section_to_remove, $end) + strlen($end)); - $string = str_replace($section_to_remove, '', $string); + // Retrieve webpage + $pageUrl = self::URI . (empty($topic) ? 'news/' : $topic . '/'); + $html = getSimpleHTMLDOM($pageUrl) + or returnServerError('Could not request CNET: ' . $pageUrl); + + // Process articles + foreach($html->find('div.assetBody, div.riverPost') as $element) { + + if(count($this->items) >= 10) { + break; } - return $string; - } + $article_title = trim($element->find('h2, h3', 0)->plaintext); + $article_uri = self::URI . substr($element->find('a', 0)->href, 1); + $article_thumbnail = $element->parent()->find('img[src]', 0)->src; + $article_timestamp = strtotime($element->find('time.assetTime, div.timeAgo', 0)->plaintext); + $article_author = trim($element->find('a[rel=author], a.name', 0)->plaintext); + $article_content = '<p><b>' . trim($element->find('p.dek', 0)->plaintext) . '</b></p>'; - function cleanArticle($article_html){ - $article_html = '<p>' . substr($article_html, strpos($article_html, '<p>') + 3); - $article_html = stripWithDelimiters($article_html, '<span class="credit">', '</span>'); - $article_html = stripWithDelimiters($article_html, '<script', '</script>'); - $article_html = stripWithDelimiters($article_html, '<div class="shortcode related-links', '</div>'); - $article_html = stripWithDelimiters($article_html, '<a class="clickToEnlarge">', '</a>'); - return $article_html; - } + if (is_null($article_thumbnail)) + $article_thumbnail = extractFromDelimiters($element->innertext, '<img src="', '"'); - $pageUrl = self::URI . (empty($this->getInput('topic')) ? '' : 'topics/' . $this->getInput('topic') . '/'); - $html = getSimpleHTMLDOM($pageUrl) or returnServerError('Could not request CNET: ' . $pageUrl); - $limit = 0; + if (!empty($article_title) && !empty($article_uri) && strpos($article_uri, self::URI . 'news/') !== false) { - foreach($html->find('div.assetBody') as $element) { - if($limit < 8) { - $article_title = trim($element->find('h2', 0)->plaintext); - $article_uri = self::URI . ($element->find('a', 0)->href); - $article_timestamp = strtotime($element->find('time.assetTime', 0)->plaintext); - $article_author = trim($element->find('a[rel=author]', 0)->plaintext); + $article_html = getSimpleHTMLDOMCached($article_uri) or $article_html = null; - if(!empty($article_title) && !empty($article_uri) && strpos($article_uri, '/news/') !== false) { - $article_html = getSimpleHTMLDOM($article_uri) - or returnServerError('Could not request CNET: ' . $article_uri); - $article_content = trim( - cleanArticle( + if (!is_null($article_html)) { + + if (empty($article_thumbnail)) + $article_thumbnail = $article_html->find('div.originalImage', 0); + if (empty($article_thumbnail)) + $article_thumbnail = $article_html->find('span.imageContainer', 0); + if (is_object($article_thumbnail)) + $article_thumbnail = $article_thumbnail->find('img', 0)->src; + + $article_content .= trim( + $this->cleanArticle( extractFromDelimiters( - $article_html, - '<div class="articleContent', - '<footer>' + $article_html, '<article', '<footer' ) ) ); - - $item = array(); - $item['uri'] = $article_uri; - $item['title'] = $article_title; - $item['author'] = $article_author; - $item['timestamp'] = $article_timestamp; - $item['content'] = $article_content; - $this->items[] = $item; - $limit++; } - } - } - } - public function getName(){ - if(!is_null($this->getInput('topic'))) { - $topic = $this->getInput('topic'); - return 'CNET News Bridge' . (empty($topic) ? '' : ' - ' . $topic); + $item = array(); + $item['uri'] = $article_uri; + $item['title'] = $article_title; + $item['author'] = $article_author; + $item['timestamp'] = $article_timestamp; + $item['enclosures'] = array($article_thumbnail); + $item['content'] = $article_content; + $this->items[] = $item; + } } - - return parent::getName(); } } diff --git a/bridges/ChristianDailyReporterBridge.php b/bridges/ChristianDailyReporterBridge.php index b8cbf3c..85f664d 100644 --- a/bridges/ChristianDailyReporterBridge.php +++ b/bridges/ChristianDailyReporterBridge.php @@ -7,6 +7,9 @@ class ChristianDailyReporterBridge extends BridgeAbstract { const DESCRIPTION = 'The Unofficial Christian Daily Reporter RSS'; // const CACHE_TIMEOUT = 86400; // 1 day + public function getIcon() { + return self::URI . 'images/cdrfavicon.png'; + } public function collectData() { $uri = 'https://www.christiandailyreporter.com/'; diff --git a/bridges/CommonDreamsBridge.php b/bridges/CommonDreamsBridge.php index e4dcb63..22b9238 100644 --- a/bridges/CommonDreamsBridge.php +++ b/bridges/CommonDreamsBridge.php @@ -3,7 +3,7 @@ class CommonDreamsBridge extends FeedExpander { const MAINTAINER = 'nyutag'; const NAME = 'CommonDreams Bridge'; - const URI = 'http://www.commondreams.org/'; + const URI = 'https://www.commondreams.org/'; const DESCRIPTION = 'Returns the newest articles.'; public function collectData(){ diff --git a/bridges/ContainerLinuxReleasesBridge.php b/bridges/ContainerLinuxReleasesBridge.php index 06c8ac4..ae43888 100644 --- a/bridges/ContainerLinuxReleasesBridge.php +++ b/bridges/ContainerLinuxReleasesBridge.php @@ -26,12 +26,16 @@ class ContainerLinuxReleasesBridge extends BridgeAbstract { ] ]; - public function getReleaseFeed($jsonUrl) { + private function getReleaseFeed($jsonUrl) { $json = getContents($jsonUrl) or returnServerError('Could not request Core OS Website.'); return json_decode($json, true); } + public function getIcon() { + return 'https://coreos.com/assets/ico/favicon.png'; + } + public function collectData() { $data = $this->getReleaseFeed($this->getJsonUri()); diff --git a/bridges/CpasbienBridge.php b/bridges/CpasbienBridge.php deleted file mode 100644 index f9b4b50..0000000 --- a/bridges/CpasbienBridge.php +++ /dev/null @@ -1,74 +0,0 @@ -<?php -class CpasbienBridge extends BridgeAbstract { - - const MAINTAINER = 'lagaisse'; - const NAME = 'Cpasbien Bridge'; - const URI = 'http://www.cpasbien.cm'; - const CACHE_TIMEOUT = 86400; // 24h - const DESCRIPTION = 'Returns latest torrents from a request query'; - - const PARAMETERS = array( array( - 'q' => array( - 'name' => 'Search', - 'required' => true, - 'title' => 'Type your search' - ) - )); - - public function collectData(){ - $request = str_replace(' ', '-', trim($this->getInput('q'))); - $html = getSimpleHTMLDOM(self::URI . '/recherche/' . urlencode($request) . '.html') - or returnServerError('No results for this query.'); - - foreach($html->find('#gauche', 0)->find('div') as $episode) { - if($episode->getAttribute('class') == 'ligne0' - || $episode->getAttribute('class') == 'ligne1') { - - $urlepisode = $episode->find('a', 0)->getAttribute('href'); - $htmlepisode = getSimpleHTMLDOMCached($urlepisode, 86400 * 366 * 30); - - $item = array(); - $item['author'] = $episode->find('a', 0)->text(); - $item['title'] = $episode->find('a', 0)->text(); - $item['pubdate'] = $this->getCachedDate($urlepisode); - $textefiche = $htmlepisode->find('#textefiche', 0)->find('p', 1); - - if(isset($textefiche)) { - $item['content'] = $textefiche->text(); - } else { - $p = $htmlepisode->find('#textefiche', 0)->find('p'); - if(!empty($p)) { - $item['content'] = $htmlepisode->find('#textefiche', 0)->find('p', 0)->text(); - } - } - - $item['id'] = $episode->find('a', 0)->getAttribute('href'); - $item['uri'] = self::URI . $htmlepisode->find('#telecharger', 0)->getAttribute('href'); - $this->items[] = $item; - } - } - } - - public function getName(){ - if(!is_null($this->getInput('q'))) { - return $this->getInput('q') . ' : ' . self::NAME; - } - - return parent::getName(); - } - - private function getCachedDate($url){ - debugMessage('getting pubdate from url ' . $url . ''); - - // Initialize cache - $cache = Cache::create('FileCache'); - $cache->setPath(CACHE_DIR . '/pages'); - - $params = [$url]; - $cache->setParameters($params); - - // Get cachefile timestamp - $time = $cache->getTime(); - return ($time !== false ? $time : time()); - } -} diff --git a/bridges/CrewbayBridge.php b/bridges/CrewbayBridge.php new file mode 100644 index 0000000..6951777 --- /dev/null +++ b/bridges/CrewbayBridge.php @@ -0,0 +1,211 @@ +<?php +class CrewbayBridge extends BridgeAbstract { + const MAINTAINER = 'couraudt'; + const NAME = 'Crewbay Bridge'; + const URI = 'https://www.crewbay.com'; + const DESCRIPTION = 'Returns the newest sailing offers.'; + const PARAMETERS = array( + array( + 'keyword' => array( + 'name' => 'Filter by keyword', + 'title' => 'Enter the keyword to filter here' + ), + 'type' => array( + 'name' => 'Type of search', + 'title' => 'Choose between finding a boat or a crew', + 'type' => 'list', + 'values' => array( + 'Find a boat' => 'boats', + 'Find a crew' => 'crew' + ) + ), + 'status' => array( + 'name' => 'Status on the boat', + 'title' => 'Choose between recreational or professional classified ads', + 'type' => 'list', + 'values' => array( + 'Recreational' => 'recreational', + 'Professional' => 'professional' + ) + ), + 'recreational_position' => array( + 'name' => 'Recreational position wanted', + 'title' => 'Filter by recreational position you wanted aboard', + 'required' => false, + 'type' => 'list', + 'values' => array( + '' => '', + 'Amateur Crew' => 'Amateur Crew', + 'Friendship' => 'Friendship', + 'Competent Crew' => 'Competent Crew', + 'Racing' => 'Racing', + 'Voluntary work' => 'Voluntary work', + 'Mile building' => 'Mile building' + ) + ), + 'professional_position' => array( + 'name' => 'Professional position wanted', + 'title' => 'Filter by professional position you wanted aboard', + 'required' => false, + 'type' => 'list', + 'values' => array( + '' => '', + '1st Engineer' => '1st Engineer', + '1st Mate' => '1st Mate', + 'Beautician' => 'Beautician', + 'Bosun' => 'Bosun', + 'Captain' => 'Captain', + 'Chef' => 'Chef', + 'Steward(ess)' => 'Steward(ess)', + 'Deckhand' => 'Deckhand', + 'Delivery Crew' => 'Delivery Crew', + 'Dive Instructor' => 'Dive Instructor', + 'Masseur' => 'Masseur', + 'Medical Staff' => 'Medical Staff', + 'Nanny' => 'Nanny', + 'Navigator' => 'Navigator', + 'Racing Crew' => 'Racing Crew', + 'Teacher' => 'Teacher', + 'Electrical Engineer' => 'Electrical Engineer', + 'Fitter' => 'Fitter', + '2nd Engineer' => '2nd Engineer', + '3rd Engineer' => '3rd Engineer', + 'Lead Deckhand' => 'Lead Deckhand', + 'Security Officer' => 'Security Officer', + 'O.O.W' => 'O.O.W', + '1st Officer' => '1st Officer', + '2nd Officer' => '2nd Officer', + '3rd Officer' => '3rd Officer', + 'Captain/Engineer' => 'Captain/Engineer', + 'Hairdresser' => 'Hairdresser', + 'Fitness Trainer' => 'Fitness Trainer', + 'Laundry' => 'Laundry', + 'Solo Steward/ess' => 'Solo Steward/ess', + 'Stew/Deck' => 'Stew/Deck', + '2nd Steward/ess' => '2nd Steward/ess', + '3rd Steward/ess' => '3rd Steward/ess', + 'Chief Steward/ess' => 'Chief Steward/ess', + 'Head Housekeeper' => 'Head Housekeeper', + 'Purser' => 'Purser', + 'Cook' => 'Cook', + 'Cook/Stew' => 'Cook/Stew', + '2nd Chef' => '2nd Chef', + 'Head Chef' => 'Head Chef', + 'Administrator' => 'Administrator', + 'P.A' => 'P.A', + 'Villa staff' => 'Villa staff', + 'Housekeeping/Stew' => 'Housekeeping/Stew', + 'Stew/Beautician' => 'Stew/Beautician', + 'Stew/Masseuse' => 'Stew/Masseuse', + 'Manager' => 'Manager', + 'Sailing instructor' => 'Sailing instructor' + ) + ) + ) + ); + + public function collectData() { + $url = $this->getURI(); + $html = getSimpleHTMLDOM($url) or returnClientError('No results for this query.'); + + $annonces = $html->find('#SearchResults div.result'); + $limit = 0; + + foreach ($annonces as $annonce) { + $detail = $annonce->find('.btn--profile', 0); + $htmlDetail = getSimpleHTMLDOMCached($detail->getAttribute('data-modal-href')); + + $item = array(); + + if ($this->getInput('type') == 'boats') { + $titleSelector = '.title h2'; + } else { + $titleSelector = '.layout__item h2'; + } + $userName = $annonce->find('.result--description a', 0)->plaintext; + $annonceTitle = trim($annonce->find($titleSelector, 0)->plaintext); + if (empty($annonceTitle)) { + $item['title'] = $userName; + } else { + $item['title'] = $userName . ' - ' . $annonceTitle; + } + + $item['uri'] = $detail->href; + $images = $annonce->find('.avatar img'); + $item['enclosures'] = array(end($images)->getAttribute('src')); + + if ($this->getInput('type') == 'boats') { + $fields = array('job', 'boat', 'skipper'); + } else { + $fields = array('profile', 'positions', 'info', 'qualifications' , 'skills', 'references'); + } + + $content = ''; + foreach ($fields as $field) { + $info = $htmlDetail->find('.profile--modal-body .info-' . $field, 0); + if ($info) { + $content .= $htmlDetail->find('.profile--modal-body .info-' . $field, 0)->innertext; + } + } + + $item['content'] = $content; + + if (!empty($this->getInput('keyword'))) { + $keyword = strtolower($this->getInput('keyword')); + if (strpos(strtolower($item['title']), $keyword) === false) { + if (strpos(strtolower($content), $keyword) === false) { + continue; + } + } + } + + if (!empty($this->getInput('recreational_position')) || !empty($this->getInput('professional_position'))) { + if ($this->getInput('type') == 'boats') { + if ($this->getInput('status') == 'professional') { + $positions = array($annonce->find('.title .position', 0)->plaintext); + } else { + $positions = array(str_replace('Wanted:', '', $annonce->find('.content li', 0)->plaintext)); + } + } else { + $positions = explode("\r\n", trim($htmlDetail->find('.info-positions .value', 0)->plaintext)); + } + + $found = false; + $keyword = $this->getInput('status') == 'professional' ? 'professional_position' : 'recreational_position'; + foreach ($positions as $position) { + if (strpos(trim($position), $this->getInput($keyword)) !== false) { + $found = true; + break; + } + } + + if (!$found) { + continue; + } + } + + $this->items[] = $item; + $limit += 1; + + if ($limit == 10) break; + } + } + + public function getURI() { + $uri = parent::getURI(); + + if ($this->getInput('type') == 'boats') { + $uri .= '/boats'; + } else { + $uri .= '/crew'; + } + + if ($this->getInput('status') == 'professional') { + $uri .= '/professional'; + } else { + $uri .= '/recreational'; + } + + return $uri; + } +} diff --git a/bridges/DailymotionBridge.php b/bridges/DailymotionBridge.php index d075041..ff8d482 100644 --- a/bridges/DailymotionBridge.php +++ b/bridges/DailymotionBridge.php @@ -48,6 +48,10 @@ class DailymotionBridge extends BridgeAbstract { return $metadata; } + public function getIcon() { + return 'https://static1-ssl.dmcdn.net/images/neon/favicons/android-icon-36x36.png.vf806ca4ed0deed812'; + } + public function collectData(){ $html = ''; $limit = 5; diff --git a/bridges/DanbooruBridge.php b/bridges/DanbooruBridge.php index 82f2167..755399f 100644 --- a/bridges/DanbooruBridge.php +++ b/bridges/DanbooruBridge.php @@ -1,7 +1,7 @@ <?php class DanbooruBridge extends BridgeAbstract { - const MAINTAINER = 'mitsukarenai'; + const MAINTAINER = 'mitsukarenai, logmanoriginal'; const NAME = 'Danbooru'; const URI = 'http://donmai.us/'; const CACHE_TIMEOUT = 1800; // 30min @@ -57,11 +57,80 @@ class DanbooruBridge extends BridgeAbstract { } public function collectData(){ - $html = getSimpleHTMLDOM($this->getFullURI()) + $content = getContents($this->getFullURI()) or returnServerError('Could not request ' . $this->getName()); + $html = Fix_Simple_Html_Dom::str_get_html($content); + foreach($html->find(static::PATHTODATA) as $element) { $this->items[] = $this->getItemFromElement($element); } } } + +/** + * This class is a monkey patch to 'extend' simplehtmldom to recognize <source> + * tags (HTML5) as self closing tag. This patch should be removed once + * simplehtmldom was fixed. This seems to be a issue with more tags: + * https://sourceforge.net/p/simplehtmldom/bugs/83/ + * + * The tag itself is valid according to Mozilla: + * + * The HTML <picture> element serves as a container for zero or more <source> + * elements and one <img> element to provide versions of an image for different + * display device scenarios. The browser will consider each of the child <source> + * elements and select one corresponding to the best match found; if no matches + * are found among the <source> elements, the file specified by the <img> + * element's src attribute is selected. The selected image is then presented in + * the space occupied by the <img> element. + * + * -- https://developer.mozilla.org/en-US/docs/Web/HTML/Element/picture + * + * Notice: This class uses parts of the original simplehtmldom, adjusted to pass + * the guidelines of RSS-Bridge (formatting) + */ +final class Fix_Simple_Html_Dom extends simple_html_dom { + + /* copy from simple_html_dom, added 'source' at the end */ + protected $self_closing_tags = array( + 'img' => 1, + 'br' => 1, + 'input' => 1, + 'meta' => 1, + 'link' => 1, + 'hr' => 1, + 'base' => 1, + 'embed' => 1, + 'spacer' => 1, + 'source' => 1 + ); + + /* copy from simplehtmldom, changed 'simple_html_dom' to 'Fix_Simple_Html_Dom' */ + public static function str_get_html($str, + $lowercase = true, + $forceTagsClosed = true, + $target_charset = DEFAULT_TARGET_CHARSET, + $stripRN = true, + $defaultBRText = DEFAULT_BR_TEXT, + $defaultSpanText = DEFAULT_SPAN_TEXT) + { + $dom = new Fix_Simple_Html_Dom(null, + $lowercase, + $forceTagsClosed, + $target_charset, + $stripRN, + $defaultBRText, + $defaultSpanText); + + if (empty($str) || strlen($str) > MAX_FILE_SIZE) { + + $dom->clear(); + return false; + + } + + $dom->load($str, $lowercase, $stripRN); + + return $dom; + } +} diff --git a/bridges/DauphineLibereBridge.php b/bridges/DauphineLibereBridge.php index 7547d74..20c8207 100644 --- a/bridges/DauphineLibereBridge.php +++ b/bridges/DauphineLibereBridge.php @@ -3,7 +3,7 @@ class DauphineLibereBridge extends FeedExpander { const MAINTAINER = 'qwertygc'; const NAME = 'Dauphine Bridge'; - const URI = 'http://www.ledauphine.com/'; + const URI = 'https://www.ledauphine.com/'; const CACHE_TIMEOUT = 7200; // 2h const DESCRIPTION = 'Returns the newest articles.'; @@ -49,8 +49,9 @@ class DauphineLibereBridge extends FeedExpander { private function extractContent($url){ $html2 = getSimpleHTMLDOMCached($url); - $text = $html2->find('div.column', 0)->innertext; - $text = preg_replace('@<script[^>]*?>.*?</script>@si', '', $text); - return $text; + foreach ($html2->find('.noprint, link, script, iframe, .shareTool, .contentInfo') as $remove) { + $remove->outertext = ''; + } + return $html2->find('div.content', 0)->innertext; } } diff --git a/bridges/DealabsBridge.php b/bridges/DealabsBridge.php index fb88f37..525cb38 100644 --- a/bridges/DealabsBridge.php +++ b/bridges/DealabsBridge.php @@ -46,23 +46,914 @@ class DealabsBridge extends PepperBridgeAbstract { 'required' => 'true', 'title' => 'Groupe dont il faut afficher les deals', 'values' => array( + 'Abonnements internet' => 'abonnements-internet', 'Accessoires & gadgets' => 'accessoires-gadgets', + 'Accessoires photo' => 'accessoires-photo', + 'Accessoires vélo' => 'accessoires-velo', + 'Acer' => 'acer', + 'Adaptateurs' => 'adaptateurs', + 'Adhérents Fnac' => 'adherents-fnac', + 'adidas' => 'adidas', + 'adidas Stan Smith' => 'adidas-stan-smith', + 'adidas Superstar' => 'adidas-superstar', + 'adidas ZX Flux' => 'adidas-zx-flux', + 'Adoucissant' => 'adoucissant', + 'Agendas' => 'agendas', + 'Age of Empires' => 'age-of-empires', + 'Alarmes' => 'alarmes', 'Alimentation & boissons' => 'alimentation-boissons', + 'Alimentation PC' => 'alimentation-pc', + 'Amazon Echo' => 'amazon-echo', + 'Amazon Fire TV' => 'amazon-fire-tv', + 'Amazon Kindle' => 'amazon-kindle', + 'Amazon Prime' => 'amazon-prime', + 'AMD Ryzen' => 'amd-ryzen', + 'AMD Vega' => 'amd-vega', + 'amiibo' => 'amiibo', + 'Amplis' => 'amplis', + 'Ampoules' => 'ampoules', 'Animaux' => 'animaux', + 'Anker' => 'anker', + 'Antivirus' => 'antivirus', + 'Antivols' => 'antivols', + 'Appareils de musculation' => 'appareils-de-musculation', + 'Appareils photo' => 'appareils-photo', + 'Apple AirPods' => 'apple-airpods', + 'Apple' => 'apple', + 'Apple iPad' => 'apple-ipad', + 'Apple iPad Mini' => 'apple-ipad-mini', + 'Apple iPad Pro' => 'apple-ipad-pro', + 'Apple iPhone 6' => 'apple-iphone-6', + 'Apple iPhone 7' => 'apple-iphone-7', + 'Apple iPhone 8' => 'apple-iphone-8', + 'Apple iPhone 8 Plus' => 'apple-iphone-8-plus', + 'Apple iPhone' => 'apple-iphone', + 'Apple iPhone SE' => 'apple-iphone-se', + 'Apple iPhone X' => 'apple-iphone-x', + 'Apple MacBook Air' => 'apple-macbook-air', + 'Apple MacBook Pro' => 'apple-macbook-pro', + 'Apple TV' => 'apple-tv', + 'Apple Watch' => 'apple-watch', + 'Applications Android' => 'applications-android', + 'Applications' => 'applications', + 'Applications iOS' => 'applications-ios', 'Applis & logiciels' => 'applis-logiciels', + 'Arbres à chat' => 'arbres-a-chat', + 'Asmodée' => 'asmodee', + 'Aspirateurs' => 'aspirateurs', + 'Aspirateurs Dyson' => 'aspirateurs-dyson', + 'Aspirateurs robot' => 'aspirateurs-robot', + 'Assassin's Creed' => 'assassin-s-creed', + 'Assassin's Creed Origins' => 'assassin-s-creed-origins', + 'Assurances' => 'assurances', + 'Asus' => 'asus', + 'ASUS Transformer' => 'asus-transformer', + 'Asus ZenFone 2' => 'asus-zenfone-2', + 'Asus ZenFone 3' => 'asus-zenfone-3', + 'Asus ZenFone 4' => 'asus-zenfone-4', + 'Asus ZenFone GO' => 'asus-zenfone-go', + 'Aukey' => 'aukey', + 'Auto' => 'auto', + 'Auto-Moto' => 'auto-moto', + 'Autoradios' => 'autoradios', + 'Baby foot' => 'baby-foot', + 'BabyLiss' => 'babyliss', + 'Babyphones' => 'babyphones', + 'Bagagerie' => 'bagagerie', + 'Balançoires' => 'balancoires', + 'Bandes dessinées' => 'bandes-dessinees', + 'Banques' => 'banques', + 'Barbecue' => 'barbecue', + 'Barbie' => 'barbie', + 'Barres de son' => 'barres-de-son', + 'Batteries externes' => 'batteries-externes', + 'Battlefield 1' => 'battlefield-1', + 'Battlefield' => 'battlefield', + 'Béaba' => 'beaba', + 'Beats by Dre' => 'beats-by-dre', + 'BenQ' => 'benq', + 'Be quiet!' => 'be-quiet', + 'Biberons' => 'biberons', + 'Bières' => 'bieres', + 'Bijoux' => 'bijoux', + 'Billets d'avion' => 'billets-d-avion', + 'BioShock' => 'bioshock', + 'BioShock Infinite' => 'bioshock-infinite', + 'Bitdefender' => 'bitdefender', + 'Blackberry' => 'blackberry', + 'Black & Decker' => 'black-decker', + 'Blédina' => 'bledina', + 'Blu-Ray' => 'blu-ray', + 'Boissons' => 'boissons', + 'Boîtes à outils' => 'boites-a-outils', + 'Boîtiers PC' => 'boitiers-pc', + 'Bonbons' => 'bonbons', + 'Borderlands' => 'borderlands', + 'Bosch' => 'bosch', + 'Bose' => 'bose', + 'Bose SoundLink' => 'bose-soundlink', + 'Bottes' => 'bottes', + 'Box beauté' => 'box-beaute', + 'Bracelet fitness' => 'bracelet-fitness', + 'Brandt' => 'brandt', + 'Braun Silk Épil' => 'braun-silk-epil', + 'Bricolage' => 'bricolage', + 'Brosses à dents' => 'brosses-a-dents', + 'Cable management' => 'cable-management', + 'Câbles' => 'cables', + 'Câbles HDMI' => 'cables-hdmi', + 'Câbles USB' => 'cables-usb', + 'Cadres' => 'cadres', + 'Café' => 'cafe', + 'Café en grain' => 'cafe-en-grain', + 'Cafetières' => 'cafetieres', + 'Cahiers' => 'cahiers', + 'Call of Duty' => 'call-of-duty', + 'Call of Duty: Infinite Warfare' => 'call-of-duty-infinite-warfare', + 'Calor' => 'calor', + 'Caméras' => 'cameras', + 'Caméras IP' => 'cameras-ip', + 'Camping' => 'camping', + 'Carburant' => 'carburant', + 'Cartables' => 'cartables', + 'Cartes graphiques' => 'cartes-graphiques', + 'Cartes mères' => 'cartes-meres', + 'Cartes postales' => 'cartes-postales', + 'Casques audio' => 'casques-audio', + 'Casques sans fil' => 'casques-sans-fil', + 'Casquettes' => 'casquettes', + 'Casseroles' => 'casseroles', + 'CDAV' => 'cdav', + 'Ceintures' => 'ceintures', + 'Chaises' => 'chaises', + 'Chaises hautes' => 'chaises-hautes', + 'Chargeurs' => 'chargeurs', + 'Chasse' => 'chasse', + 'Chats' => 'chats', + 'Chaussons' => 'chaussons', + 'Chaussures adidas' => 'chaussures-adidas', + 'Chaussures' => 'chaussures', + 'Chaussures de football' => 'chaussures-de-football', + 'Chaussures de randonnée' => 'chaussures-de-randonnee', + 'Chaussures de running' => 'chaussures-de-running', + 'Chaussures de ski' => 'chaussures-de-ski', + 'Chaussures de ville' => 'chaussures-de-ville', + 'Chaussures Nike' => 'chaussures-nike', + 'Chelsea boots' => 'chelsea-boots', + 'Chemises' => 'chemises', + 'Chiens' => 'chiens', + 'Chocolat' => 'chocolat', + 'Chuck Taylor' => 'chuck-taylor', + 'Cinéma' => 'cinema', + 'Civilization' => 'civilization', + 'Civilization VI' => 'civilization-vi', + 'Clarks' => 'clarks', + 'Claviers' => 'claviers', + 'Claviers gamer' => 'claviers-gamer', + 'Claviers mécaniques' => 'claviers-mecaniques', + 'Clés USB' => 'cles-usb', + 'Composteurs' => 'composteurs', + 'Concerts' => 'concerts', + 'Congélateurs' => 'congelateurs', + 'Consoles' => 'consoles', 'Consoles & jeux vidéo' => 'consoles-jeux-video', + 'Converse' => 'converse', + 'Costumes' => 'costumes', + 'Couches' => 'couches', + 'Couettes' => 'couettes', + 'Couteaux de cuisine' => 'couteaux-de-cuisine', + 'Couverts' => 'couverts', + 'Covoiturage' => 'covoiturage', + 'Crédits' => 'credits', + 'Croquettes pour chien' => 'croquettes-pour-chien', + 'Cuisinières' => 'cuisinieres', 'Culture & divertissement' => 'culture-divertissement', + 'Cyclisme' => 'cyclisme', + 'DDR3' => 'ddr3', + 'DDR4' => 'ddr4', + 'Décoration' => 'decoration', + 'Deezer' => 'deezer', + 'Dell' => 'dell', + 'Delsey' => 'delsey', + 'Denon' => 'denon', + 'Dentifrices' => 'dentifrices', + 'Destiny 2' => 'destiny-2', + 'Destiny' => 'destiny', + 'Dishonored' => 'dishonored', + 'Disneyland Paris' => 'disneyland-paris', + 'Disques durs externes' => 'disques-durs-externes', + 'Disques durs internes' => 'disques-durs', + 'DJI' => 'dji', + 'Dosettes Nespresso' => 'dosettes-nespresso', + 'Dosettes Senseo' => 'dosettes-senseo', + 'Dosettes Tassimo' => 'dosettes-tassimo', + 'Draisiennes' => 'draisiennes', + 'Drones' => 'drones', + 'Durex' => 'durex', + 'DVD' => 'dvd', + 'Dyson' => 'dyson', + 'Eastpak' => 'eastpak', + 'ebooks' => 'ebooks', + 'Écharpes & foulards' => 'echarpes-et-foulards', + 'Écouteurs' => 'ecouteurs', + 'Écouteurs intra-auriculaires' => 'ecouteurs-intra-auriculaires', + 'Écouteurs sans fil' => 'ecouteurs-sans-fil', + 'Écouteurs sport' => 'ecouteurs-sport', + 'Écrans 21" et moins' => 'ecrans-21-pouces-et-moins', + 'Écrans 24"' => 'ecrans-24-pouces', + 'Écrans 27"' => 'ecrans-27-pouces', + 'Écrans 29" et plus' => 'ecrans-29-pouces-et-plus', + 'Écrans 4K / UHD' => 'ecrans-4k-uhd', + 'Écrans Acer' => 'ecrans-acer', + 'Écrans Asus' => 'ecrans-asus', + 'Écrans BenQ' => 'ecrans-benq', + 'Écrans Dell' => 'ecrans-dell', + 'Écrans de projection' => 'ecrans-de-projection', + 'Écrans' => 'ecrans', + 'Écrans FreeSync' => 'ecrans-freesync', + 'Écrans gamer' => 'ecrans-gamer', + 'Écrans incurvés' => 'ecrans-incurves', + 'Écrans Philips' => 'ecrans-philips', + 'Écrans Samsung' => 'ecrans-samsung', + 'Électricité (matériel)' => 'electricite', + 'Electrolux' => 'electrolux', + 'Électroménager' => 'electromenager', + 'Embauchoirs' => 'embauchoirs', + 'Enceintes Bluetooth' => 'enceintes-bluetooth', + 'Enceintes' => 'enceintes', + 'Engrais' => 'engrais', + 'Entretien du jardin' => 'entretien-du-jardin', + 'Épicerie' => 'epicerie', + 'Épilateurs à lumière pulsée' => 'epilateurs-a-lumiere-pulsee', + 'Épilateurs électriques' => 'epilateurs-electriques', + 'Épilation' => 'epilation', + 'Équipement auto' => 'equipement-auto', + 'Équipement motard' => 'equipement-motard', + 'Équipement sportif' => 'equipement-sportif', + 'Érotisme' => 'erotisme', + 'Escarpins' => 'escarpins', + 'Événements sportifs' => 'evenements-sportifs', + 'Expositions' => 'expositions', + 'F1 2017' => 'f1-2017', + 'Facom' => 'facom', + 'Fallout 4' => 'fallout-4', + 'Fallout' => 'fallout', + 'Fards à paupières' => 'fards-a-paupieres', + 'Fast-foods' => 'fast-foods', + 'Fauteuils' => 'fauteuils', + 'Fers à lisser / à friser' => 'fers-a-lisser-a-friser', + 'Fers à souder' => 'fers-a-souder', + 'Festivals' => 'festivals', + 'Feutres' => 'feutres', + 'FIFA 17' => 'fifa-17', + 'FIFA 18' => 'fifa-18', + 'FIFA 19' => 'fifa-19', + 'FIFA' => 'fifa', + 'Figurines' => 'figurines', + 'Films' => 'films', + 'Final Fantasy' => 'final-fantasy', + 'Final Fantasy XII' => 'final-fantasy-xii', + 'fitbit' => 'fitbit', + 'Flash' => 'flash', + 'Fluval' => 'fluval', + 'Foires & salons' => 'foires-et-salons', + 'Fonds de teint' => 'fonds-de-teint', + 'Football' => 'football', + 'Forfaits mobiles' => 'forfaits-mobiles', + 'For Honor' => 'for-honor', + 'Formule 1' => 'formule-1', + 'Fortnite' => 'fortnite', + 'Forza Horizon 3' => 'forza-horizon-3', + 'Forza Motorsport 7' => 'forza-motorsport-7', + 'Fossil' => 'fossil', + 'Fournitures de bureau' => 'fournitures-de-bureau', + 'Fournitures scolaires' => 'fournitures-scolaires', + 'Fours à poser' => 'fours-a-poser', + 'Fours encastrables' => 'fours-encastrables', + 'Fours' => 'fours', + 'Friandises pour chat' => 'friandises-pour-chat', + 'Friandises pour chien' => 'friandises-pour-chien', + 'Friskies' => 'friskies', + 'Fruits & légumes' => 'fruits-et-legumes', + 'FURminator' => 'furminator', + 'Futuroscope' => 'futuroscope', + 'Gamelles' => 'gamelles', + 'Game of Thrones' => 'game-of-thrones', + 'Gants' => 'gants', + 'Gants moto' => 'gants-moto', + 'Garmin' => 'garmin', + 'Gâteaux & biscuits' => 'gateaux-et-biscuits', + 'Gels douche' => 'gels-douche', + 'Geox' => 'geox', + 'Gigoteuses' => 'gigoteuses', + 'Gillette' => 'gillette', + 'Glaces' => 'glaces', + 'God of War' => 'god-of-war', + 'Google Chromecast' => 'google-chromecast', + 'Google Home' => 'google-home', + 'Google Pixel 2' => 'google-pixel-2', + 'Google Pixel 2 XL' => 'google-pixel-2-xl', + 'Google Pixel' => 'google-pixel', + 'Google Pixel XL' => 'google-pixel-xl', + 'GoPro Hero' => 'gopro-hero', + 'Gran Turismo' => 'gran-turismo', 'Gratuit' => 'gratuit', + 'Grille-pain' => 'grille-pain', + 'GTA' => 'gta', + 'GTA V' => 'gta-v', + 'Guitares' => 'guitares', + 'Gyropodes' => 'gyropodes', + 'Haltères & poids' => 'halteres-et-poids', + 'Hamacs' => 'hamacs', + 'Hama' => 'hama', + 'Hand spinners' => 'hand-spinners', + 'Harnais pour chien' => 'harnais-pour-chien', + 'Harry Potter' => 'harry-potter', + 'Havaianas' => 'havaianas', + 'HDD' => 'hdd', + 'Hisense' => 'hisense', + 'Home Cinéma' => 'home-cinema', + 'Honor 6X' => 'honor-6x', + 'Honor 8' => 'honor-8', + 'Honor 8 Pro' => 'honor-8-pro', + 'Honor 9' => 'honor-9', + 'Horizon Zero Dawn' => 'horizon-zero-dawn', + 'Hôtels' => 'hotels', + 'Hoverboards' => 'hoverboards', + 'HTC 10' => 'htc-10', + 'HTC Desire' => 'htc-desire', + 'HTC One M9' => 'htc-one-m9', + 'HTC U11' => 'htc-u11', + 'HTC U Play' => 'htc-u-play', + 'HTC U Ultra' => 'htc-u-ultra', + 'HTC Vive' => 'htc-vive', + 'Huawei Mate 10' => 'huawei-mate-10', + 'Huawei Mate 9' => 'huawei-mate-9', + 'Huawei P10' => 'huawei-p10', + 'Huawei P10 Lite' => 'huawei-p10-lite', + 'Huawei P10 Plus' => 'huawei-p10-plus', + 'Huawei P20' => 'huawei-p20', + 'Huawei P20 Pro' => 'huawei-p20-pro', + 'Huawei P8 Lite' => 'huawei-p8-lite', + 'Huawei P9 Lite' => 'huawei-p9-lite', + 'Hubs' => 'hubs', + 'Huile moteur' => 'huile-moteur', + 'Hygiène corporelle' => 'hygiene-corporelle', + 'Hygiène de la maison' => 'hygiene-de-la-maison', + 'Hygiène des bébés' => 'hygiene-des-bebes', 'Image, son & vidéo' => 'image-son-video', + 'Impressions photo' => 'impressions-photo', + 'Imprimantes 3D' => 'imprimantes-3d', + 'Imprimantes Brother' => 'imprimantes-brother', + 'Imprimantes Canon' => 'imprimantes-canon', + 'Imprimantes Epson' => 'imprimantes-epson', + 'Imprimantes HP' => 'imprimantes-hp', + 'Imprimantes' => 'imprimantes', + 'Imprimantes laser' => 'imprimantes-laser', + 'Imprimantes multifonctions' => 'imprimantes-multifonctions', 'Informatique' => 'informatique', + 'Instruments de musique' => 'instruments-de-musique', + 'Intel i5' => 'intel-i5', + 'Intel i7' => 'intel-i7', + 'JBL Flip' => 'jbl-flip', + 'JBL' => 'jbl', + 'Jeans' => 'jeans', + 'Jeux d'apprentissage' => 'jeux-d-apprentissage', + 'Jeux d'extérieur' => 'jeux-d-exterieur', + 'Jeux d'imitation' => 'jeux-d-imitation', + 'Jeux de construction' => 'jeux-de-construction', + 'Jeux de société' => 'jeux-de-societe', 'Jeux & jouets' => 'jeux-jouets', - 'Maison & jardin' => 'maison-jardin', + 'Jeux Nintendo Switch' => 'jeux-nintendo-switch', + 'Jeux & paris' => 'jeux-et-paris', + 'Jeux PC dématérialisés' => 'jeux-pc-dematerialises', + 'Jeux PlayStation 4' => 'jeux-playstation-4', + 'Jeux pour bébés' => 'jeux-pour-bebes', + 'Jeux PS4 dématérialisés' => 'jeux-ps4-dematerialises', + 'Jeux PS Plus' => 'jeux-ps-plus', + 'Jeux vidéo' => 'jeux-video', + 'Jeux Wii U' => 'jeux-wii-u', + 'Jeux Xbox dématérialisés' => 'jeux-xbox-dematerialises', + 'Jeux Xbox One' => 'jeux-xbox-one', + 'Jeux Xbox with Gold' => 'jeux-xbox-with-gold', + 'Journaux numériques' => 'journaux-numeriques', + 'Journaux papier' => 'journaux-papier', + 'Joy-Con' => 'manettes-nintendo-switch-joy-con', + 'Jungle Speed' => 'jungle-speed', + 'Kaspersky' => 'kaspersky', + 'Kinder' => 'kinder', + 'Kindle Paperwhite' => 'kindle-paperwhite', + 'Kindle Voyage' => 'kindle-voyage', + 'Kobo Aura 2' => 'kobo-aura-2', + 'Kobo Aura H2o' => 'kobo-aura-h2o', + 'Kobo' => 'kobo', + 'L'annale du destin' => 'l-annale-du-destin', + 'L'ombre de la guerre' => 'l-ombre-de-la-guerre', + 'L'ombre du Mordor' => 'l-ombre-du-mordor', + 'Lacoste' => 'lacoste', + 'Lapeyre' => 'lapeyre', + 'La Terre du Milieu' => 'la-terre-du-milieu', + 'Lavage auto' => 'lavage-auto', + 'Lave-linge frontal' => 'lave-linge-frontal', + 'Lave-linge' => 'lave-linge', + 'Lave-linge séchant' => 'lave-linge-sechant', + 'Lave-linge top' => 'lave-linge-top', + 'Lave-vaisselle' => 'lave-vaisselle', + 'Le bâton de la vérité' => 'le-baton-de-la-verite', + 'Lecteurs Blu-Ray' => 'lecteurs-blu-ray', + 'Lecteurs CD' => 'lecteurs-cd', + 'Lecteurs DVD' => 'lecteurs-dvd', + 'Lego' => 'lego', + 'Lego Star Wars' => 'lego-star-wars', + 'Lenovo K6 Note' => 'lenovo-k6-note', + 'Lenovo' => 'lenovo', + 'Lenovo P8' => 'lenovo-p8', + 'Lenovo Tab 3' => 'lenovo-tab-3', + 'Lenovo Tab 4' => 'lenovo-tab-4', + 'Lenovo Yoga' => 'lenovo-yoga', + 'Lenovo Yoga Tab 3' => 'lenovo-yoga-tab-3', + 'Lentilles de contact' => 'lentilles-de-contact', + 'Le Seigneur des anneaux' => 'le-seigneur-des-anneaux', + 'Les Sims' => 'les-sims', + 'Lessive' => 'lessive', + 'Levi's' => 'levi-s', + 'LG G4' => 'lg-g4', + 'LG G5' => 'lg-g5', + 'LG G6' => 'lg-g6', + 'LG' => 'lg', + 'LG OLED TV' => 'lg-oled-tv', + 'LG Q6' => 'lg-q6', + 'LG Q8' => 'lg-q8', + 'Life is Strange' => 'life-is-strange', + 'Linge de maison' => 'linge-de-maison', + 'Lingerie' => 'lingerie', + 'Lingettes pour bébés' => 'lingettes-pour-bebes', + 'Liseuses' => 'liseuses', + 'Litière pour chat' => 'litiere-pour-chat', + 'Lits' => 'lits', + 'Lits pour bébé' => 'lits-pour-bebe', + 'Livres audio' => 'livres-audio', + 'Livres' => 'livres', + 'Livres photo' => 'livres-photo', + 'Location de voiture' => 'location-de-voiture', + 'Logiciels de sécurité' => 'logiciels-de-securite', + 'Logiciels Microsoft' => 'logiciels-microsoft', + 'Logitech Harmony' => 'logitech-harmony', + 'Logitech' => 'logitech', + 'Loup-Garou' => 'loup-garou', + 'Lubrifiants' => 'lubrifiants', + 'Luminaires' => 'luminaires', + 'Lunettes de natation' => 'lunettes-de-natation', + 'Lunettes de soleil' => 'lunettes-de-soleil', + 'MacBook' => 'macbook', + 'Mac de bureau' => 'mac-de-bureau', + 'Machines à café à dosettes' => 'machines-a-cafe-a-dosettes', + 'Machines à café en grain' => 'machines-a-cafe-en-grain', + 'Machines à pain' => 'machines-a-pain', + 'Machines Dolce Gusto' => 'machines-dolce-gusto', + 'Machines Nespresso' => 'machines-nespresso', + 'Machines Senseo' => 'machines-senseo', + 'Magasins d'usine' => 'magasins-usine', + 'Magazines' => 'magazines', + 'Maillots de bain' => 'maillots-de-bain', + 'Maillots de football' => 'maillots-de-football', + 'Maison & Jardin' => 'maison-et-jardin', + 'Makita' => 'makita', + 'Manettes Nintendo Switch Pro' => 'manettes-nintendo-switch-pro', + 'Manettes PlayStation 4' => 'manettes-playstation-4', + 'Manettes Xbox One Elite' => 'manettes-xbox-one-elite', + 'Manettes Xbox One' => 'manettes-xbox-one', + 'Manix' => 'manix', + 'Manteaux' => 'manteaux', + 'Maquillage' => 'maquillage', + 'Mario Kart' => 'mario-kart', + 'Marteaux & maillets' => 'marteaux-et-maillets', + 'Mascara' => 'mascara', + 'Masques de ski' => 'masques-de-ski', + 'Mass Effect: Andromeda' => 'mass-effect-andromeda', + 'Matchs de football' => 'matchs-de-football', + 'Matelas gonflables' => 'matelas-gonflables', + 'Matelas' => 'matelas', + 'Matériaux de construction' => 'materiaux-de-construction', + 'Matériel de ski' => 'materiel-de-ski', + 'Medion' => 'medion', + 'Meubles pour chat' => 'meubles-pour-chat', + 'Micro-casques gaming' => 'micro-casques-gaming', + 'Micro-ondes' => 'micro-ondes', + 'Microphones' => 'microphones', + 'Micro-SD' => 'micro-sd', + 'Microsoft Office' => 'microsoft-office', + 'Microsoft Surface' => 'microsoft-surface', + 'Miele' => 'miele', + 'Minecraft' => 'minecraft', + 'Mixeurs' => 'mixeurs', + 'M&M's' => 'metm-s', + 'Mobilier' => 'mobilier', 'Mode & accessoires' => 'mode-accessoires', - 'Santé & cosmétiques' => 'hygiene-sante-cosmetiques', + 'Mode enfants' => 'mode-enfants', + 'Mode femme' => 'mode-femme', + 'Mode homme' => 'mode-homme', + 'Modélisme' => 'modelisme', + 'Monopoly' => 'monopoly', + 'Montage PC' => 'montage-pc', + 'Montres' => 'montres', + 'Moto C Plus' => 'moto-c-plus', + 'Moto E4' => 'moto-e4', + 'Moto G5' => 'moto-g5', + 'Moto G5 Plus' => 'moto-g5-plus', + 'Moto G5S' => 'moto-g5s', + 'Moto G5S Plus' => 'moto-g5s-plus', + 'Moto M' => 'moto-m', + 'Moto' => 'moto', + 'Moto Z2' => 'moto-z2', + 'Moto Z2 Play' => 'moto-z2-play', + 'Moulinex' => 'moulinex', + 'Mousses à raser' => 'mousses-a-raser', + 'MSI' => 'msi', + 'Musées' => 'musees', + 'Musique' => 'musique', + 'NAS' => 'nas', + 'Natation' => 'natation', + 'Navigation' => 'navigation', + 'NERF' => 'nerf', + 'New Balance' => 'new-balance', + 'Nike Air Force' => 'nike-air-force', + 'Nike Air Max' => 'nike-air-max', + 'Nike Free' => 'nike-free', + 'Nike Huarache' => 'nike-huarache', + 'Nike' => 'nike', + 'Nintendo Classic Mini' => 'nintendo-classic-mini', + 'Nintendo' => 'nintendo', + 'Nintendo Switch' => 'nintendo-switch', + 'Nivea' => 'nivea', + 'Nokia 5' => 'nokia-5', + 'Nokia 6' => 'nokia-6', + 'Nokia 8' => 'nokia-8', + 'Nourriture pour chat' => 'nourriture-pour-chat', + 'Nourriture pour chien' => 'nourriture-pour-chien', + 'Nutella' => 'nutella', + 'Nvidia GeForce GTX 1060' => 'nvidia-geforce-gtx-1060', + 'Nvidia GeForce GTX 1070' => 'nvidia-geforce-gtx-1070', + 'Nvidia GeForce GTX 1080' => 'nvidia-geforce-gtx-1080', + 'Nvidia GeForce GTX 1080 Ti' => 'nvidia-geforce-gtx-1080-ti', + 'Nvidia' => 'nvidia', + 'Nvidia Shield' => 'nvidia-shield', + 'Objectifs' => 'objectifs', + 'Oculus Rift' => 'oculus-rift', + 'Oiseaux' => 'oiseaux', + 'OnePlus 5' => 'oneplus-5', + 'OnePlus 5T' => 'oneplus-5t', + 'OnePlus 6' => 'oneplus-6', + 'Onkyo' => 'onkyo', + 'Ordinateurs de bureau' => 'ordinateurs-de-bureau', + 'Oreillers' => 'oreillers', + 'Outillage' => 'outillage', + 'Outils de jardinage' => 'outils-de-jardinage', + 'Overwatch' => 'overwatch', + 'Packs clavier-souris' => 'packs-clavier-souris', + 'Paiement en ligne' => 'paiement-en-ligne', + 'Pampers' => 'pampers', + 'Panasonic' => 'panasonic', + 'Panier Plus' => 'panier-plus', + 'Pantalons' => 'pantalons', + 'Papeterie' => 'papeterie', + 'Papier peint' => 'papier-peint', + 'Papier toilette' => 'papier-toilette', + 'Parapharmacie' => 'parapharmacie', + 'Parc Astérix' => 'parc-asterix', + 'Parfums femme' => 'parfums-femme', + 'Parfums homme' => 'parfums-homme', + 'Parfums' => 'parfums', + 'Parkas' => 'parkas', + 'Parrot' => 'parrot', + 'Partitions' => 'partitions', + 'PC de bureau complets' => 'pc-de-bureau-complets', + 'PC gamer complets' => 'pc-gamer-complets', + 'PC hybrides' => 'hybrides', + 'PC portables' => 'pc-portables', + 'Pêche' => 'peche', + 'Peintures' => 'peintures', + 'Peluches' => 'peluches', + 'Perceuses' => 'perceuses', + 'Périphériques PC' => 'peripheriques-pc', + 'Pèse-personnes' => 'pese-personnes', + 'PES' => 'pro-evolution-soccer', + 'Petites voitures' => 'petites-voitures', + 'Philips Hue' => 'philips-hue', + 'Philips Lumea' => 'philips-lumea', + 'Philips One Blade' => 'philips-one-blade', + 'Philips' => 'philips', + 'Philips Sonicare' => 'philips-sonicare', + 'Photo' => 'photo', + 'Pièces auto' => 'pieces-auto', + 'Pièces moto' => 'pieces-moto', + 'Pièces vélo' => 'pieces-velo', + 'Piles' => 'piles', + 'Piles rechargeables' => 'piles-rechargeables', + 'Pinces' => 'pinces', + 'Pizza' => 'pizza', + 'Places de cinéma' => 'places-de-cinema', + 'Plage' => 'plage', + 'Plantes' => 'plantes', + 'Plaques de cuisson' => 'plaques-de-cuisson', + 'Platines vinyle' => 'platines-vinyle', + 'Playmobil' => 'playmobil', + 'PlayStation 4' => 'playstation-4', + 'PlayStation 4 Pro' => 'playstation-4-pro', + 'PlayStation 4 Slim' => 'playstation-4-slim', + 'PlayStation' => 'playstation', + 'PlayStation Plus' => 'playstation-plus', + 'Playstation Store' => 'playstation-store', + 'Plomberie' => 'plomberie', + 'Pneus' => 'pneus', + 'PocketBook' => 'pocketbook', + 'Poêles' => 'poeles', + 'Pokémon' => 'pokemon', + 'Portables gamer' => 'portables-gamer', + 'Porte-bébé' => 'porte-bebe', + 'Portefeuilles' => 'portefeuilles', + 'Posters' => 'posters', + 'Potager' => 'potager', + 'Poulaillers' => 'poulaillers', + 'Poupées' => 'poupees', + 'Poussettes' => 'poussettes', + 'Premiers secours' => 'premiers-secours', + 'Préservatifs' => 'preservatifs', + 'Princesse Tam-Tam' => 'princesse-tam-tam', + 'Processeurs' => 'processeurs', + 'Protection de la maison' => 'protection-de-la-maison', + 'Protections intimes' => 'protections-intimes', + 'Puériculture' => 'puericulture', + 'Pulls' => 'pulls', + 'Puma' => 'puma', + 'Purificateurs d'air' => 'purificateurs-d-air', + 'Purina' => 'purina', + 'Puzzles' => 'puzzles', + 'Pyjamas pour bébés' => 'pyjamas-pour-bebes', + 'Pyjamas' => 'pyjamas', + 'Qobuz' => 'qobuz', + 'RAM' => 'ram', + 'Randonnée' => 'randonnee', + 'Rasage' => 'rasage', + 'Rasoirs électriques' => 'rasoirs-electriques', + 'Rasoirs manuels' => 'rasoirs-manuels', + 'Raspberry Pi' => 'raspberry-pi', + 'Ray-Ban' => 'ray-ban', + 'Razer' => 'razer', + 'Réductions étudiants & jeunes' => 'reductions-etudiants-et-jeunes', + 'Reebok' => 'reebok', + 'Réfrigérateurs' => 'refrigerateurs', + 'Réhausseurs' => 'rehausseurs', + 'Remington' => 'remington', + 'Répéteurs' => 'repeteurs', + 'Réseau' => 'reseau', + 'Resident Evil 7' => 'resident-evil-7', + 'Resident Evil' => 'resident-evil', + 'Restaurants' => 'restaurants', + 'Richelieus' => 'richelieus', + 'Risk' => 'risk', + 'Rongeurs' => 'rongeurs', + 'Rouges à lèvres' => 'rouges-a-levres', + 'Routeurs' => 'routeurs', + 'Royal Canin' => 'royal-canin', + 'Running' => 'running', + 'Sacs à dos' => 'sacs-a-dos', + 'Sacs à langer' => 'sacs-a-langer', + 'Sacs à main' => 'sacs-a-main', + 'Samsonite' => 'samsonite', + 'Samsung Galaxy A5' => 'samsung-galaxy-a5', + 'Samsung Galaxy Note 8' => 'samsung-galaxy-note-8', + 'Samsung Galaxy S7 Edge' => 'samsung-galaxy-s7-edge', + 'Samsung Galaxy S7' => 'samsung-galaxy-s7', + 'Samsung Galaxy S8' => 'samsung-galaxy-s8', + 'Samsung Galaxy S8+' => 'samsung-galaxy-s8plus', + 'Samsung Galaxy S9' => 'samsung-galaxy-s9', + 'Samsung Galaxy Tab A' => 'samsung-galaxy-tab-a', + 'Samsung Galaxy Tab S2' => 'samsung-galaxy-tab-s2', + 'Samsung Galaxy Tab S3' => 'samsung-galaxy-tab-s3', + 'Samsung Gear' => 'samsung-gear', + 'Samsung Gear VR' => 'samsung-gear-vr', + 'Samsung' => 'samsung', + 'Sandales' => 'sandales', + 'SanDisk' => 'sandisk', + 'Santé & Cosmétiques' => 'sante-et-cosmetiques', + 'Savons' => 'savons', + 'Scanners' => 'scanners', + 'Scies' => 'scies', + 'Scooters' => 'scooters', + 'Seagate' => 'seagate', + 'Sécateurs' => 'secateurs', + 'Sèche-cheveux' => 'seche-cheveux', + 'Sèche-linge' => 'seche-linge', + 'Séjours' => 'sejours', + 'Sennheiser' => 'sennheiser', + 'Séries TV' => 'series-tv', 'Services divers' => 'services-divers', + 'Serviettes hygiéniques' => 'serviettes-hygieniques', + 'Serviettes' => 'serviettes', + 'Sextoys' => 'sextoys', + 'Shorts de bain' => 'shorts-de-bain', + 'Shorts' => 'shorts', + 'Sièges auto' => 'sieges-auto', + 'Siemens' => 'siemens', + 'Skechers' => 'sketchers', + 'Ski' => 'ski', + 'Skyrim' => 'skyrim', + 'Smartbox' => 'smartbox', + 'Smart Home' => 'smart-home', + 'Smartphones à moins de 100€' => 'smartphones-moins-de-100', + 'Smartphones à moins de 200€' => 'smartphones-moins-de-200', + 'Smartphones Android' => 'smartphones-android', + 'Smartphones Huawei' => 'smartphones-huawei', + 'Smartphones Nokia' => 'smartphones-nokia', + 'Smartphones Samsung' => 'smartphones-samsung', + 'Smartphones' => 'smartphones', + 'Smartphones Xiaomi' => 'smartphones-xiaomi', + 'Smart TV' => 'smart-tv', + 'Smartwatch' => 'smartwatch', + 'Sneakers' => 'sneakers', + 'Soin des cheveux' => 'soin-des-cheveux', + 'Sonos PLAYBAR' => 'sonos-playbar', + 'Sonos' => 'sonos', + 'Sony PlayStation VR' => 'sony-playstation-vr', + 'Sony' => 'sony', + 'Sony Xperia XA1' => 'sony-xperia-xa1', + 'Sony Xperia X Compact' => 'sony-xperia-x-compact', + 'Sony Xperia XZ1 Compact' => 'sony-xperia-xz1-compact', + 'Sony Xperia XZ1' => 'sony-xperia-xz1', + 'Sony Xperia XZ Premium' => 'sony-xperia-xz-premium', + 'Sony Xperia Z3' => 'sony-xperia-z3', + 'Sorties' => 'sorties', + 'Souris gamer' => 'souris-gamer', + 'Souris Logitech' => 'souris-logitech', + 'Souris sans fil' => 'souris-sans-fil', + 'Souris' => 'souris', + 'South Park' => 'south-park', + 'Spectacles comiques' => 'spectacles-comiques', + 'Spectacles' => 'spectacles', 'Sports & plein air' => 'sports-plein-air', + 'Spotify' => 'spotify', + 'SSD' => 'ssd', + 'Star Wars Battlefront' => 'star-wars-battlefront', + 'Stickers muraux' => 'stickers-muraux', + 'Stihl' => 'stihl', + 'Stockage externe' => 'stockage', + 'Streaming musical' => 'streaming-musical', + 'Stylos' => 'stylos', + 'Sucettes' => 'sucettes', + 'Super Mario' => 'super-mario', + 'Support GPS & smartphone' => 'support-gps-et-smartphone', + 'Surface Pro 4' => 'surface-pro-4', + 'Surgelés' => 'surgeles', + 'Surveillance' => 'surveillance', + 'Swatch' => 'swatch', + 'Switch réseau' => 'switch-reseau', + 'Systèmes d'exploitation' => 'systemes-d-exploitation', + 'Systèmes multiroom' => 'systemes-multiroom', + 'Tables à langer' => 'tables-a-langer', + 'Tables de camping' => 'tables-de-camping', + 'Tables de mixage' => 'tables-de-mixage', + 'Tables' => 'tables', + 'Tablettes graphiques Huion' => 'huion', + 'Tablettes graphiques' => 'tablettes-graphiques', + 'Tablettes graphiques Wacom' => 'wacom', + 'Tablettes Lenovo' => 'tablettes-lenovo', + 'Tablettes Samsung' => 'tablettes-samsung', + 'Tablettes' => 'tablettes', + 'Tablettes Xiaomi' => 'tablettes-xiaomi', + 'Tampons' => 'tampons', + 'Tapis' => 'tapis', + 'Taxis' => 'taxis', + 'Tefal' => 'tefal', + 'Télécommandes' => 'telecommandes', + 'Téléphones fixes' => 'telephones-fixes', 'Téléphonie' => 'telephonie', - 'Voyages & sorties' => 'voyages-sorties-restaurants', + 'Téléviseurs' => 'televiseurs', + 'Tentes' => 'tentes', + 'Têtes de brosse à dents de rechange' => 'tetes-de-brosse-a-dents-de-rechange', + 'Théâtre' => 'theatre', + 'The Legend of Zelda' => 'the-legend-of-zelda', + 'Thermomètres' => 'thermometres', + 'Thermomix' => 'thermomix', + 'Thés glacés' => 'thes-glaces', + 'Thés' => 'thes', + 'The Walking dead' => 'the-walking-dead', + 'The Witcher 3' => 'the-witcher-3', + 'The Witcher' => 'the-witcher', + 'Time's Up!' => 'time-s-up', + 'Tom Clancy's Ghost Recon: Wildlands' => 'tom-clancy-s-ghost-recon-wildlands', + 'Tom Clancy's The Division' => 'tom-clancy-s-the-division', + 'Tom Clancy's' => 'tom-clancy-s', + 'TomTom' => 'tomtom', + 'Tondeuses à gazon' => 'tondeuses-a-gazon', + 'Tondeuses' => 'tondeuses', + 'Toner' => 'toner', + 'Torchons' => 'torchons', + 'Toshiba' => 'toshiba', + 'Total War' => 'total-war', + 'Total War: Warhammer II' => 'total-war-warhammer-ii', + 'Total War: Warhammer' => 'total-war-warhammer', + 'Tournevis & visseuses' => 'tournevis-et-visseuses', + 'TP-Link' => 'tp-link', + 'Transats & cosys' => 'transats-et-cosys', + 'Transports en commun' => 'transports-en-commun', + 'Trixie' => 'trixie', + 'Tronçonneuses' => 'tronconneuses', + 'Trottinettes électriques' => 'trottinettes-electriques', + 'Trottinettes' => 'trottinettes', + 'T-shirts' => 't-shirts', + 'TV 39'' et moins' => 'tv-39-pouces-et-moins', + 'TV 40'' à 64''' => 'tv-40-pouces-a-64-pouces', + 'TV 4K' => 'tv-4k', + 'TV 65'' et plus' => 'tv-65-pouces-et-plus', + 'TV Full HD' => 'tv-full-hd', + 'TV incurvées' => 'tv-incurvees', + 'TV LG' => 'tv-lg', + 'TV OLED' => 'tv-oled', + 'TV Panasonic' => 'tv-panasonic', + 'TV Philips' => 'tv-philips', + 'TV Samsung' => 'tv-samsung', + 'TV Sony' => 'tv-sony', + 'Ultraportables' => 'ultraportables', + 'Uncharted 4' => 'uncharted-4', + 'Uncharted: The Lost Legacy' => 'uncharted-the-lost-legacy', + 'Uncharted' => 'uncharted', + 'Ustensiles de cuisine' => 'ustensiles-de-cuisine', + 'Ustensiles de cuisson' => 'ustensiles-de-cuisson', + 'Vaisselle' => 'vaisselle', + 'Valises cabine' => 'valises-cabine', + 'Valises rigides' => 'valises-rigides', + 'Valises' => 'valises', + 'Variétés & revues' => 'varietes-et-revues', + 'Vases' => 'vases', + 'Veet' => 'veet', + 'Vélos d'appartement' => 'velos-d-appartement', + 'Vélos' => 'velos', + 'Ventilateurs' => 'ventilateurs', + 'Ventirad' => 'ventirad', + 'Vernis à ongles' => 'vernis-a-ongles', + 'Vestes' => 'vestes', + 'Vêtements d'été' => 'vetements-d-ete', + 'Vêtements d'hiver' => 'vetements-d-hiver', + 'Vêtements de grossesse' => 'vetements-de-grossesse', + 'Vêtements de ski' => 'vetements-de-ski', + 'Vêtements de sport' => 'vetements-de-sport', + 'Vêtements pour bébé' => 'vetements-pour-bebe', + 'Vêtements techniques' => 'vetements-techniques', + 'Vidéoprojecteurs 3D' => 'videoprojecteurs-3d', + 'Vidéoprojecteurs Acer' => 'videoprojecteurs-acer', + 'Vidéoprojecteurs BenQ' => 'videoprojecteurs-benq', + 'Vidéoprojecteurs Epson' => 'videoprojecteurs-epson', + 'Vidéoprojecteurs HD' => 'videoprojecteurs-hd', + 'Vidéoprojecteurs LG' => 'videoprojecteurs-lg', + 'Vidéoprojecteurs Optoma' => 'videoprojecteurs-optoma', + 'Vidéoprojecteurs' => 'projecteurs', + 'Vidéo' => 'video', + 'Vins' => 'vins', + 'Visites & patrimoine' => 'visites-et-patrimoine', + 'VOD' => 'vod', + 'Voitures télécommandées' => 'voitures-telecommandees', + 'Voyages & sorties' => 'voyages-et-sorties', + 'Voyages' => 'voyages', + 'VPN' => 'vpn', + 'VR' => 'vr', + 'VTC' => 'vtc', + 'VTT' => 'vtt', + 'Wacom Cintiq' => 'cintiq', + 'Watercooling' => 'watercooling', + 'WD (Western Digital)' => 'western-digital', + 'Wearables' => 'wearables', + 'Whey' => 'whey', + 'Whirlpool' => 'whirlpool', + 'Whiskas' => 'whiskas', + 'Wii U' => 'wii-u', + 'Wiko' => 'wiko', + 'Windows' => 'windows', + 'WindScribe' => 'windscribe', + 'Wolfenstein II: The New Colossus' => 'wolfenstein-ii-the-new-colossus', + 'Wolfenstein' => 'wolfenstein', + 'Wonderbox' => 'wonderbox', + 'Xbox Live' => 'xbox-live', + 'Xbox One S' => 'xbox-one-s', + 'Xbox One' => 'xbox-one', + 'Xbox One X' => 'xbox-one-x', + 'Xbox' => 'xbox', + 'Xiaomi Mi6' => 'xiaomi-mi6', + 'Xiaomi Mi A1' => 'xiaomi-mi-a1', + 'Xiaomi Mi Band' => 'xiaomi-mi-band', + 'Xiaomi Mi Box' => 'xiaomi-mi-box', + 'Xiaomi Mi Max' => 'xiaomi-mi-max', + 'Xiaomi Mi Mix 2' => 'xiaomi-mi-mix-2', + 'Xiaomi Mi Mix' => 'xiaomi-mi-mix', + 'Xiaomi Mi Pad 3' => 'xiaomi-mi-pad-3', + 'Xiaomi Redmi 4A' => 'xiaomi-redmi-4a', + 'Xiaomi Redmi 4X' => 'xiaomi-redmi-4x', + 'Xiaomi Redmi Note 4' => 'xiaomi-redmi-note-4', + 'Xiaomi Smart Home' => 'xiaomi-smart-home', + 'Xiaomi' => 'xiaomi', + 'Yamaha' => 'yamaha', + 'Zelda: Breath of the Wild' => 'zelda-breath-of-the-wild', + 'Zoos' => 'zoos', ) ), 'order' => array( @@ -157,7 +1048,7 @@ class PepperBridgeAbstract extends BridgeAbstract { /** * Get the Deal data from the choosen group in the choosed order */ - public function collectDataGroup() + protected function collectDataGroup() { $group = $this->getInput('group'); @@ -171,7 +1062,7 @@ class PepperBridgeAbstract extends BridgeAbstract { /** * Get the Deal data from the choosen keywords and parameters */ - public function collectDataKeywords() + protected function collectDataKeywords() { $q = $this->getInput('q'); $hide_expired = $this->getInput('hide_expired'); @@ -183,10 +1074,10 @@ class PepperBridgeAbstract extends BridgeAbstract { $url = $this->i8n('bridge-uri') . '/search/advanced?q=' . urlencode($q) - . '&hide_expired='. $hide_expired - . '&hide_local='. $hide_local - . '&priceFrom='. $priceFrom - . '&priceTo='. $priceTo + . '&hide_expired=' . $hide_expired + . '&hide_local=' . $hide_local + . '&priceFrom=' . $priceFrom + . '&priceTo=' . $priceTo /* Some default parameters * search_fields : Search in Titres & Descriptions & Codes * sort_by : Sort the search by new deals @@ -199,7 +1090,7 @@ class PepperBridgeAbstract extends BridgeAbstract { /** * Get the Deal data using the given URL */ - public function collectDeals($url){ + protected function collectDeals($url){ $html = getSimpleHTMLDOM($url) or returnServerError($this->i8n('request-error')); $list = $html->find('article[id]'); @@ -229,10 +1120,8 @@ class PepperBridgeAbstract extends BridgeAbstract { $selectorHot = implode( ' ', /* Notice this is a space! */ array( - 'flex', - 'flex--align-c', - 'flex--justify-space-between', - 'space--b-2', + 'cept-vote-box', + 'vote-box' ) ); @@ -241,9 +1130,7 @@ class PepperBridgeAbstract extends BridgeAbstract { ' ', /* Notice this is a space! */ array( 'cept-description-container', - 'overflow--wrap-break', - 'size--all-s', - 'size--fromW3-m' + 'overflow--wrap-break' ) ); @@ -253,8 +1140,7 @@ class PepperBridgeAbstract extends BridgeAbstract { array( 'size--all-s', 'flex', - 'flex--justify-e', - 'flex--grow-1', + 'boxAlign-jc--all-fe' ) ); @@ -266,29 +1152,30 @@ class PepperBridgeAbstract extends BridgeAbstract { foreach ($list as $deal) { $item = array(); $item['uri'] = $deal->find('div[class=threadGrid-title]', 0)->find('a', 0)->href; - $item['title'] = $deal->find('a[class*='. $selectorLink .']', 0 + $item['title'] = $deal->find('a[class*=' . $selectorLink . ']', 0 )->plaintext; $item['author'] = $deal->find('span.thread-username', 0)->plaintext; $item['content'] = '<table><tr><td><a href="' . $deal->find( - 'a[class*='. $selectorImageLink .']', 0)->href + 'a[class*=' . $selectorImageLink . ']', 0)->href . '"><img src="' . $this->getImage($deal) . '"/></td><td><h2><a href="' - . $deal->find('a[class*='. $selectorLink .']', 0)->href + . $deal->find('a[class*=' . $selectorLink . ']', 0)->href . '">' - . $deal->find('a[class*='. $selectorLink .']', 0)->innertext + . $deal->find('a[class*=' . $selectorLink . ']', 0)->innertext . '</a></h2>' . $this->getPrice($deal) . $this->getDiscount($deal) . $this->getShipsFrom($deal) . $this->getShippingCost($deal) . $this->GetSource($deal) - . $deal->find('div[class*='. $selectorDescription .']', 0)->innertext + . $deal->find('div[class*=' . $selectorDescription . ']', 0)->innertext . '</td><td>' - . $deal->find('div[class='. $selectorHot .']', 0)->children(0)->outertext + . $deal->find('div[class*=' . $selectorHot . ']', 0) + ->find('span', 1)->outertext . '</td></table>'; - $dealDateDiv = $deal->find('div[class*='. $selectorDate .']', 0) + $dealDateDiv = $deal->find('div[class*=' . $selectorDate . ']', 0) ->find('span[class=hide--toW3]'); $itemDate = end($dealDateDiv)->plaintext; // In case of a Local deal, there is no date, but we can use @@ -327,7 +1214,7 @@ class PepperBridgeAbstract extends BridgeAbstract { { if ($deal->find( 'span[class*=thread-price]', 0) != null) { - return '<div>'.$this->i8n('price') .' : ' + return '<div>' . $this->i8n('price') . ' : ' . $deal->find( 'span[class*=thread-price]', 0 )->plaintext @@ -346,11 +1233,11 @@ class PepperBridgeAbstract extends BridgeAbstract { { if ($deal->find('span[class*=cept-shipping-price]', 0) != null) { if ($deal->find('span[class*=cept-shipping-price]', 0)->children(0) != null) { - return '<div>'. $this->i8n('shipping') .' : ' + return '<div>' . $this->i8n('shipping') . ' : ' . $deal->find('span[class*=cept-shipping-price]', 0)->children(0)->innertext . '</div>'; } else { - return '<div>'. $this->i8n('shipping') .' : ' + return '<div>' . $this->i8n('shipping') . ' : ' . $deal->find('span[class*=cept-shipping-price]', 0)->innertext . '</div>'; } @@ -366,7 +1253,7 @@ class PepperBridgeAbstract extends BridgeAbstract { private function GetSource($deal) { if ($deal->find('a[class=text--color-greyShade]', 0) != null) { - return '<div>'. $this->i8n('origin') .' : ' + return '<div>' . $this->i8n('origin') . ' : ' . $deal->find('a[class=text--color-greyShade]', 0)->outertext . '</div>'; } else { @@ -387,7 +1274,7 @@ class PepperBridgeAbstract extends BridgeAbstract { } else { $discount = ''; } - return '<div>'. $this->i8n('discount') .' : <span style="text-decoration: line-through;">' + return '<div>' . $this->i8n('discount') . ' : <span style="text-decoration: line-through;">' . $deal->find( 'span[class*=mute--text text--lineThrough]', 0 )->plaintext @@ -428,13 +1315,13 @@ class PepperBridgeAbstract extends BridgeAbstract { 'cept-thread-img' ) ); - if ($deal->find('img[class='. $selectorLazy .']', 0) != null) { + if ($deal->find('img[class=' . $selectorLazy . ']', 0) != null) { return json_decode( html_entity_decode( - $deal->find('img[class='. $selectorLazy .']', 0) + $deal->find('img[class=' . $selectorLazy . ']', 0) ->getAttribute('data-lazy-img')))->{'src'}; } else { - return $deal->find('img[class*='. $selectorPlain .']', 0 )->src; + return $deal->find('img[class*=' . $selectorPlain . ']', 0 )->src; } } @@ -453,9 +1340,9 @@ class PepperBridgeAbstract extends BridgeAbstract { 'text--color-greyShade' ) ); - if ($deal->find('span[class='. $selector .']', 0) != null) { + if ($deal->find('span[class=' . $selector . ']', 0) != null) { return '<div>' - . $deal->find('span[class='. $selector .']', 0)->children(2)->plaintext + . $deal->find('span[class=' . $selector . ']', 0)->children(2)->plaintext . '</div>'; } else { return ''; @@ -558,12 +1445,12 @@ class PepperBridgeAbstract extends BridgeAbstract { public function getName(){ switch($this->queriedContext) { case $this->i8n('context-keyword'): - return $this->i8n('bridge-name') . ' - '. $this->i8n('title-keyword') .' : '. $this->getInput('q'); + return $this->i8n('bridge-name') . ' - ' . $this->i8n('title-keyword') . ' : ' . $this->getInput('q'); break; case $this->i8n('context-group'): $values = $this->getParameters()[$this->i8n('context-group')]['group']['values']; $group = array_search($this->getInput('group'), $values); - return $this->i8n('bridge-name') . ' - '. $this->i8n('title-group'). ' : '. $group; + return $this->i8n('bridge-name') . ' - ' . $this->i8n('title-group') . ' : ' . $group; break; default: // Return default value return static::NAME; @@ -577,7 +1464,7 @@ class PepperBridgeAbstract extends BridgeAbstract { * the "$lang" class variable in the local class * @return various the local content needed */ - public function i8n($key) + protected function i8n($key) { if (array_key_exists($key, $this->lang)) { return $this->lang[$key]; diff --git a/bridges/DesoutterBridge.php b/bridges/DesoutterBridge.php new file mode 100644 index 0000000..94059f6 --- /dev/null +++ b/bridges/DesoutterBridge.php @@ -0,0 +1,240 @@ +<?php +class DesoutterBridge extends BridgeAbstract { + + const CATEGORY_NEWS = 'News & Events'; + const CATEGORY_INDUSTRY = 'Industry 4.0 News'; + + const NAME = 'Desoutter Bridge'; + const URI = 'https://www.desouttertools.com'; + const DESCRIPTION = 'Returns feeds for news from Desoutter'; + const MAINTAINER = 'logmanoriginal'; + const CACHE_TIMEOUT = 86400; // 24 hours + + const PARAMETERS = array( + self::CATEGORY_NEWS => array( + 'news_lang' => array( + 'name' => 'Language', + 'type' => 'list', + 'required' => true, + 'title' => 'Select your language', + 'defaultValue' => 'Corporate', + 'values' => array( + 'Corporate' + => 'https://www.desouttertools.com/about-desoutter/news-events', + 'Česko' + => 'https://www.desouttertools.cz/o-desoutter/aktuality-udalsoti', + 'Deutschland' + => 'https://www.desoutter.de/ueber-desoutter/news-events', + 'España' + => 'https://www.desouttertools.es/sobre-desoutter/noticias-eventos', + 'México' + => 'https://www.desouttertools.mx/acerca-desoutter/noticias-eventos', + 'France' + => 'https://www.desouttertools.fr/a-propos-de-desoutter/actualites-evenements', + 'Magyarország' + => 'https://www.desouttertools.hu/a-desoutter-vallalatrol/hirek-esemenyek', + 'Italia' + => 'https://www.desouttertools.it/su-desoutter/news-eventi', + '日本' + => 'https://www.desouttertools.jp/desotanituite/niyusu-ibento', + '대한민국' + => 'https://www.desouttertools.co.kr/desoteoe-daehaeseo/nyuseu-mic-ibenteu', + 'Polska' + => 'https://www.desouttertools.pl/o-desoutter/aktualnosci-wydarzenia', + 'Brasil' + => 'https://www.desouttertools.com.br/sobre-desoutter/noti%C2%ADcias-eventos', + 'Portugal' + => 'https://www.desouttertools.pt/sobre-desoutter/notIcias-eventos', + 'România' + => 'https://www.desouttertools.ro/despre-desoutter/noutati-evenimente', + 'Российская Федерация' + => 'https://www.desouttertools.com.ru/o-desoutter/novosti-mieropriiatiia', + 'Slovensko' + => 'https://www.desouttertools.sk/o-spolocnosti-desoutter/novinky-udalosti', + 'Slovenija' + => 'https://www.desouttertools.si/o-druzbi-desoutter/novice-dogodki', + 'Sverige' + => 'https://www.desouttertools.se/om-desoutter/nyheter-evenemang', + 'Türkiye' + => 'https://www.desoutter.com.tr/desoutter-hakkinda/haberler-etkinlikler', + '中国' + => 'https://www.desouttertools.com.cn/guan-yu-ma-tou/xin-wen-he-huo-dong', + ) + ), + ), + self::CATEGORY_INDUSTRY => array( + 'industry_lang' => array( + 'name' => 'Language', + 'type' => 'list', + 'required' => true, + 'title' => 'Select your language', + 'defaultValue' => 'Corporate', + 'values' => array( + 'Corporate' + => 'https://www.desouttertools.com/industry-4-0/news', + 'Česko' + => 'https://www.desouttertools.cz/prumysl-4-0/novinky', + 'Deutschland' + => 'https://www.desoutter.de/industrie-4-0/news', + 'España' + => 'https://www.desouttertools.es/industria-4-0/noticias', + 'México' + => 'https://www.desouttertools.mx/industria-4-0/noticias', + 'France' + => 'https://www.desouttertools.fr/industrie-4-0/actualites', + 'Magyarország' + => 'https://www.desouttertools.hu/industry-4-0/hirek', + 'Italia' + => 'https://www.desouttertools.it/industry-4-0/news', + '日本' + => 'https://www.desouttertools.jp/industry-4-0/news', + '대한민국' + => 'https://www.desouttertools.co.kr/industry-4-0/news', + 'Polska' + => 'https://www.desouttertools.pl/przemysl-4-0/wiadomosci', + 'Brasil' + => 'https://www.desouttertools.com.br/industria-4-0/noticias', + 'Portugal' + => 'https://www.desouttertools.pt/industria-4-0/noticias', + 'România' + => 'https://www.desouttertools.ro/industry-4-0/noutati', + 'Российская Федерация' + => 'https://www.desouttertools.com.ru/industry-4-0/news', + 'Slovensko' + => 'https://www.desouttertools.sk/priemysel-4-0/novinky', + 'Slovenija' + => 'https://www.desouttertools.si/industrija-4-0/novice', + 'Sverige' + => 'https://www.desouttertools.se/industri-4-0/nyheter', + 'Türkiye' + => 'https://www.desoutter.com.tr/endustri-4-0/haberler', + '中国' + => 'https://www.desouttertools.com.cn/industry-4-0/news', + ) + ), + ), + 'global' => array( + 'full' => array( + 'name' => 'Load full articles', + 'type' => 'checkbox', + 'required' => false, + 'title' => 'Enable to load the full article for each item' + ) + ) + ); + + private $title; + + public function getURI() { + switch($this->queriedContext) { + case self::CATEGORY_NEWS: + return $this->getInput('news_lang') ?: parent::getURI(); + case self::CATEGORY_INDUSTRY: + return $this->getInput('industry_lang') ?: parent::getURI(); + } + + return parent::getURI(); + } + + public function getName() { + return isset($this->title) ? $this->title . ' - ' . parent::getName() : parent::getName(); + } + + public function collectData() { + + // Uncomment to generate list of languages automtically (dev mode) + /* + switch($this->queriedContext) { + case self::CATEGORY_NEWS: + $this->extractNewsLanguages(); die; + case self::CATEGORY_INDUSTRY: + $this->extractIndustryLanguages(); die; + } + */ + + $html = getSimpleHTMLDOM($this->getURI()) + or returnServerError('Could not request ' . $this->getURI()); + + $html = defaultLinkTo($html, $this->getURI()); + + $this->title = html_entity_decode($html->find('title', 0)->plaintext, ENT_QUOTES); + + foreach($html->find('article') as $article) { + $item = array(); + + $item['uri'] = $article->find('[itemprop="name"]', 0)->href; + $item['title'] = $article->find('[itemprop="name"]', 0)->title; + + if($this->getInput('full')) { + $item['content'] = $this->getFullNewsArticle($item['uri']); + } else { + $item['content'] = $article->find('[itemprop="description"]', 0)->plaintext; + } + + $this->items[] = $item; + } + + } + + private function getFullNewsArticle($uri) { + $html = getSimpleHTMLDOMCached($uri) + or returnServerError('Unable to load full article!'); + + $html = defaultLinkTo($html, $this->getURI()); + + return $html->find('section.article', 0); + } + + /** + * Generates a HTML page with a PHP formatted array of languages, + * pointing to the corresponding news pages. Implementation is based + * on the 'Corporate' site. + * @return void + */ + private function extractNewsLanguages() { + $html = getSimpleHTMLDOMCached('https://www.desouttertools.com/about-desoutter/news-events') + or returnServerError('Error loading news!'); + + $html = defaultLinkTo($html, static::URI); + + $items = $html->find('ul[class="dropdown-menu"] li'); + + $list = "\t'Corporate'\n\t=> 'https://www.desouttertools.com/about-desoutter/news-events',\n"; + + foreach($items as $item) { + $lang = trim($item->plaintext); + $uri = $item->find('a', 0)->href; + + $list .= "\t'{$lang}'\n\t=> '{$uri}',\n"; + } + + echo $list; + } + + /** + * Generates a HTML page with a PHP formatted array of languages, + * pointing to the corresponding news pages. Implementation is based + * on the 'Corporate' site. + * @return void + */ + private function extractIndustryLanguages() { + $html = getSimpleHTMLDOMCached('https://www.desouttertools.com/industry-4-0/news') + or returnServerError('Error loading news!'); + + $html = defaultLinkTo($html, static::URI); + + $items = $html->find('ul[class="dropdown-menu"] li'); + + $list = "\t'Corporate'\n\t=> 'https://www.desouttertools.com/industry-4-0/news',\n"; + + foreach($items as $item) { + $lang = trim($item->plaintext); + $uri = $item->find('a', 0)->href; + + $list .= "\t'{$lang}'\n\t=> '{$uri}',\n"; + } + + echo $list; + } + +} diff --git a/bridges/DevToBridge.php b/bridges/DevToBridge.php new file mode 100644 index 0000000..2dfc033 --- /dev/null +++ b/bridges/DevToBridge.php @@ -0,0 +1,105 @@ +<?php +class DevToBridge extends BridgeAbstract { + + const CONTEXT_BY_TAG = 'By tag'; + + const NAME = 'dev.to Bridge'; + const URI = 'https://dev.to'; + const DESCRIPTION = 'Returns feeds for tags'; + const MAINTAINER = 'logmanoriginal'; + const CACHE_TIMEOUT = 10800; // 15 min. + + const PARAMETERS = array( + self::CONTEXT_BY_TAG => array( + 'tag' => array( + 'name' => 'Tag', + 'type' => 'text', + 'required' => true, + 'title' => 'Insert your tag', + 'exampleValue' => 'python' + ), + 'full' => array( + 'name' => 'Full article', + 'type' => 'checkbox', + 'required' => false, + 'title' => 'Enable to receive the full article for each item', + 'defaultValue' => false + ) + ) + ); + + public function getURI() { + switch($this->queriedContext) { + case self::CONTEXT_BY_TAG: + if($tag = $this->getInput('tag')) { + return static::URI . '/t/' . urlencode($tag); + } + break; + } + + return parent::getURI(); + } + + public function getIcon() { + return 'https://practicaldev-herokuapp-com.freetls.fastly.net/assets/ +apple-icon-5c6fa9f2bce280428589c6195b7f1924206a53b782b371cfe2d02da932c8c173.png'; + } + + public function collectData() { + + $html = getSimpleHTMLDOMCached($this->getURI()) + or returnServerError('Could not request ' . $this->getURI()); + + $html = defaultLinkTo($html, static::URI); + + $articles = $html->find('div[class="single-article"]') + or returnServerError('Could not find articles!'); + + foreach($articles as $article) { + + if($article->find('[class*="cta"]', 0)) { // Skip ads + continue; + } + + $item = array(); + + $item['uri'] = $article->find('a[id*=article-link]', 0)->href; + $item['title'] = $article->find('h3', 0)->plaintext; + + // i.e. "Charlie Harrington・Sep 21" + $item['timestamp'] = strtotime(explode('・', $article->find('h4 a', 0)->plaintext, 2)[1]); + $item['author'] = explode('・', $article->find('h4 a', 0)->plaintext, 2)[0]; + + // Profile image + $item['enclosures'] = array($article->find('img', 0)->src); + + if($this->getInput('full')) { + $fullArticle = $this->getFullArticle($item['uri']); + $item['content'] = <<<EOD +<img src="{$item['enclosures'][0]}" alt="{$item['author']}"> +<p>{$fullArticle}</p> +EOD; + } else { + $item['content'] = <<<EOD +<img src="{$item['enclosures'][0]}" alt="{$item['author']}"> +<p>{$item['title']}</p> +EOD; + } + + $item['categories'] = array_map(function($e){ return $e->plaintext; }, $article->find('div.tags span.tag')); + + $this->items[] = $item; + } + + } + + private function getFullArticle($url) { + $html = getSimpleHTMLDOMCached($url) + or returnServerError('Unable to load article from "' . $url . '"!'); + + $html = defaultLinkTo($html, static::URI); + + return $html->find('[id="article-body"]', 0); + } + +} diff --git a/bridges/DiceBridge.php b/bridges/DiceBridge.php index dc6ea15..11218df 100644 --- a/bridges/DiceBridge.php +++ b/bridges/DiceBridge.php @@ -75,6 +75,10 @@ class DiceBridge extends BridgeAbstract { ), )); + public function getIcon() { + return 'https://assets.dice.com/techpro/img/favicons/favicon.ico'; + } + public function collectData() { $uri = 'https://www.dice.com/jobs/advancedResult.html'; $uri .= '?for_one=' . urlencode($this->getInput('for_one')); diff --git a/bridges/DilbertBridge.php b/bridges/DilbertBridge.php index 959a91a..a84e5e8 100644 --- a/bridges/DilbertBridge.php +++ b/bridges/DilbertBridge.php @@ -9,8 +9,8 @@ class DilbertBridge extends BridgeAbstract { public function collectData(){ - $html = getSimpleHTMLDOM($this->getURI()) - or returnServerError('Could not request Dilbert: ' . $this->getURI()); + $html = getSimpleHTMLDOM(self::URI) + or returnServerError('Could not request Dilbert: ' . self::URI); foreach($html->find('section.comic-item') as $element) { diff --git a/bridges/DiscogsBridge.php b/bridges/DiscogsBridge.php index 9fe4f51..534ac08 100644 --- a/bridges/DiscogsBridge.php +++ b/bridges/DiscogsBridge.php @@ -81,7 +81,7 @@ class DiscogsBridge extends BridgeAbstract { . $this->getInput('username_folder') . '/collection/folders/' . $this->getInput('folderid') - .'/releases?sort=added&sort_order=desc') + . '/releases?sort=added&sort_order=desc') or returnServerError('Unable to query discogs !'); $jsonData = json_decode($data, true)['releases']; } diff --git a/bridges/DribbbleBridge.php b/bridges/DribbbleBridge.php index 07c4c6e..5058da6 100644 --- a/bridges/DribbbleBridge.php +++ b/bridges/DribbbleBridge.php @@ -7,6 +7,11 @@ class DribbbleBridge extends BridgeAbstract { const CACHE_TIMEOUT = 1800; const DESCRIPTION = 'Returns the newest popular shots from Dribbble.'; + public function getIcon() { + return 'https://cdn.dribbble.com/assets/ +favicon-63b2904a073c89b52b19aa08cebc16a154bcf83fee8ecc6439968b1e6db569c7.ico'; + } + public function collectData(){ $html = getSimpleHTMLDOM(self::URI . '/shots') or returnServerError('Error while downloading the website content'); diff --git a/bridges/ETTVBridge.php b/bridges/ETTVBridge.php index ab90bf7..c348ca0 100644 --- a/bridges/ETTVBridge.php +++ b/bridges/ETTVBridge.php @@ -94,17 +94,20 @@ class ETTVBridge extends BridgeAbstract { ) )); + protected $results_link; + public function collectData(){ - // No control on inputs, because all have defaultValue set + // No control on inputs, because all defaultValue are set $query_str = 'torrents-search.php'; - $query_str .= '?search=' . urlencode('+'.str_replace(' ', ' +', $this->getInput('query'))); + $query_str .= '?search=' . urlencode('+' . str_replace(' ', ' +', $this->getInput('query'))); $query_str .= '&cat=' . $this->getInput('cat'); - $query_str .= 'incldead&=' . $this->getInput('status'); + $query_str .= '&incldead=' . $this->getInput('status'); $query_str .= '&lang=' . $this->getInput('lang'); $query_str .= '&sort=id&order=desc'; // Get results page - $html = getSimpleHTMLDOM(self::URI . $query_str) + $this->results_link = self::URI . $query_str; + $html = getSimpleHTMLDOM($this->results_link) or returnServerError('Could not request ' . $this->getName()); // Loop on each entry @@ -125,7 +128,7 @@ class ETTVBridge extends BridgeAbstract { $item = array(); $item['author'] = $details->children(6)->children(1)->plaintext; $item['title'] = $entry->title; - $item['uri'] = $dllinks->children(0)->children(0)->children(0)->href; + $item['uri'] = $link; $item['timestamp'] = strtotime($details->children(7)->children(1)->plaintext); $item['content'] = ''; $item['content'] .= '<br/><b>Name: </b>' . $details->children(0)->children(1)->innertext; @@ -139,4 +142,20 @@ class ETTVBridge extends BridgeAbstract { $this->items[] = $item; } } + + public function getName(){ + if($this->getInput('query')) { + return '[' . self::NAME . '] ' . $this->getInput('query'); + } + + return self::NAME; + } + + public function getURI(){ + if(isset($this->results_link) && !empty($this->results_link)) { + return $this->results_link; + } + + return self::URI; + } } diff --git a/bridges/EliteDangerousGalnetBridge.php b/bridges/EliteDangerousGalnetBridge.php index 86a1bbf..d19b360 100644 --- a/bridges/EliteDangerousGalnetBridge.php +++ b/bridges/EliteDangerousGalnetBridge.php @@ -7,6 +7,11 @@ class EliteDangerousGalnetBridge extends BridgeAbstract { const CACHE_TIMEOUT = 7200; // 2h const DESCRIPTION = 'Returns the latest page of news from Galnet'; + public function getIcon() { + return 'https://community.elitedangerous.com/sites/ +EDSITE_COMM/themes/bootstrap/bootstrap_community/favicon.ico'; + } + public function collectData(){ $html = getSimpleHTMLDOM(self::URI) or returnServerError('Error while downloading the website content'); diff --git a/bridges/ElloBridge.php b/bridges/ElloBridge.php index c0e266e..b51fed7 100644 --- a/bridges/ElloBridge.php +++ b/bridges/ElloBridge.php @@ -45,9 +45,10 @@ class ElloBridge extends BridgeAbstract { $item = array(); $item['author'] = $this->getUsername($post, $postData); $item['timestamp'] = strtotime($post->created_at); - $item['title'] = $this->findText($post->summary); + $item['title'] = strip_tags($this->findText($post->summary)); $item['content'] = $this->getPostContent($post->body); $item['enclosures'] = $this->getEnclosures($post, $postData); + $item['uri'] = self::URI . $item['author'] . '/post/' . $post->token; $content = $post->body; $this->items[] = $item; @@ -57,7 +58,7 @@ class ElloBridge extends BridgeAbstract { } - public function findText($path) { + private function findText($path) { foreach($path as $summaryElement) { @@ -71,7 +72,7 @@ class ElloBridge extends BridgeAbstract { } - public function getPostContent($path) { + private function getPostContent($path) { $content = ''; foreach($path as $summaryElement) { @@ -92,7 +93,7 @@ class ElloBridge extends BridgeAbstract { } - public function getEnclosures($post, $postData) { + private function getEnclosures($post, $postData) { $assets = []; foreach($post->links->assets as $asset) { @@ -108,7 +109,7 @@ class ElloBridge extends BridgeAbstract { } - public function getUsername($post, $postData) { + private function getUsername($post, $postData) { foreach($postData->linked->users as $user) { if($user->id == $post->links->author->id) { @@ -118,9 +119,9 @@ class ElloBridge extends BridgeAbstract { } - public function getAPIKey() { + private function getAPIKey() { $cache = Cache::create('FileCache'); - $cache->setPath(CACHE_DIR); + $cache->setPath(PATH_CACHE); $cache->setParameters(['key']); $key = $cache->loadData(); diff --git a/bridges/ElsevierBridge.php b/bridges/ElsevierBridge.php index f6ba7dd..080fe00 100644 --- a/bridges/ElsevierBridge.php +++ b/bridges/ElsevierBridge.php @@ -57,6 +57,10 @@ class ElsevierBridge extends BridgeAbstract { return ''; } + public function getIcon() { + return 'https://cdn.elsevier.io/verona/includes/favicons/favicon-32x32.png'; + } + public function collectData(){ $uri = self::URI . $this->getInput('j') . '/recent-articles/'; $html = getSimpleHTMLDOM($uri) diff --git a/bridges/EstCeQuonMetEnProdBridge.php b/bridges/EstCeQuonMetEnProdBridge.php index db9d1d5..4439d69 100644 --- a/bridges/EstCeQuonMetEnProdBridge.php +++ b/bridges/EstCeQuonMetEnProdBridge.php @@ -7,19 +7,9 @@ class EstCeQuonMetEnProdBridge extends BridgeAbstract { const CACHE_TIMEOUT = 21600; // 6h const DESCRIPTION = 'Should we put a website in production today? (French)'; - public function collectData(){ - function extractFromDelimiters($string, $start, $end){ - if(strpos($string, $start) !== false) { - $section_retrieved = substr($string, strpos($string, $start) + strlen($start)); - $section_retrieved = substr($section_retrieved, 0, strpos($section_retrieved, $end)); - return $section_retrieved; - } - - return false; - } - - $html = getSimpleHTMLDOM($this->getURI()) - or returnServerError('Could not request EstCeQuonMetEnProd: ' . $this->getURI()); + public function collectData() { + $html = getSimpleHTMLDOM(self::URI) + or returnServerError('Could not request EstCeQuonMetEnProd: ' . self::URI); $item = array(); $item['uri'] = $this->getURI() . '#' . date('Y-m-d'); @@ -28,8 +18,8 @@ class EstCeQuonMetEnProdBridge extends BridgeAbstract { $item['timestamp'] = strtotime('today midnight'); $item['content'] = str_replace( 'src="/', - 'src="' . $this->getURI(), - trim(extractFromDelimiters($html->outertext, '<body role="document">', '<br /><br />')) + 'src="' . self::URI, + trim(extractFromDelimiters($html->outertext, '<body role="document">', '<div id="share')) ); $this->items[] = $item; diff --git a/bridges/EtsyBridge.php b/bridges/EtsyBridge.php index 311d910..2671797 100644 --- a/bridges/EtsyBridge.php +++ b/bridges/EtsyBridge.php @@ -17,7 +17,7 @@ class EtsyBridge extends BridgeAbstract { 'queryextension' => array( 'name' => 'Query extension', 'type' => 'text', - 'requied' => false, + 'required' => false, 'title' => 'Insert additional query parts here (anything after ?search=<your search query>)', 'exampleValue' => '&explicit=1&locationQuery=2921044' @@ -25,9 +25,9 @@ class EtsyBridge extends BridgeAbstract { 'showimage' => array( 'name' => 'Show image in content', 'type' => 'checkbox', - 'requrired' => false, + 'required' => false, 'title' => 'Activate to show the image in the content', - 'defaultValue' => false + 'defaultValue' => 'checked' ) ) ); @@ -36,26 +36,27 @@ class EtsyBridge extends BridgeAbstract { $html = getSimpleHTMLDOM($this->getURI()) or returnServerError('Failed to receive ' . $this->getURI()); - $results = $html->find('div.block-grid-item'); + $results = $html->find('li.block-grid-item'); foreach($results as $result) { // Skip banner cards (ads for categories) - if($result->find('a.banner-card')) + if($result->find('span.ad-indicator')) continue; $item = array(); $item['title'] = $result->find('a', 0)->title; $item['uri'] = $result->find('a', 0)->href; - $item['author'] = $result->find('div.card-shop-name', 0)->plaintext; + $item['author'] = $result->find('p.text-gray-lighter', 0)->plaintext; $item['content'] = '<p>' - . $result->find('div.card-price', 0)->plaintext + . $result->find('span.currency-value', 0)->plaintext . ' ' + . $result->find('span.currency-symbol', 0)->plaintext . '</p><p>' - . $result->find('div.card-title', 0)->plaintext + . $result->find('a', 0)->title . '</p>'; - $image = $result->find('img.placeholder', 0)->src; + $image = $result->find('img.display-block', 0)->src; if($this->getInput('showimage')) { $item['content'] .= '<img src="' . $image . '">'; diff --git a/bridges/ExtremeDownloadBridge.php b/bridges/ExtremeDownloadBridge.php new file mode 100644 index 0000000..eac27e3 --- /dev/null +++ b/bridges/ExtremeDownloadBridge.php @@ -0,0 +1,104 @@ +<?php +class ExtremeDownloadBridge extends BridgeAbstract { + const NAME = 'Extreme Download'; + const URI = 'https://ww1.extreme-d0wn.com/'; + const DESCRIPTION = 'Suivi de série sur Extreme Download'; + const MAINTAINER = 'sysadminstory'; + const PARAMETERS = array( + 'Suivre la publication des épisodes d\'une série en cours de diffusion' => array( + 'url' => array( + 'name' => 'URL de la série', + 'type' => 'text', + 'required' => true, + 'title' => 'URL d\'une série sans le https://ww1.extreme-d0wn.com/', + 'exampleValue' => 'series-hd/hd-series-vostfr/46631-halt-and-catch-fire-saison-04-vostfr-hdtv-720p.html'), + 'filter' => array( + 'name' => 'Type de contenu', + 'type' => 'list', + 'required' => 'true', + 'title' => 'Type de contenu à suivre : Téléchargement, Streaming ou les deux', + 'values' => array( + 'Streaming et Téléchargement' => 'both', + 'Téléchargement' => 'download', + 'Streaming' => 'streaming' + ) + ) + ) + ); + + public function collectData(){ + $html = getSimpleHTMLDOM(self::URI . $this->getInput('url')) + or returnServerError('Could not request Extreme Download.'); + + $filter = $this->getInput('filter'); + + $typesText = array( + 'download' => 'Téléchargement', + 'streaming' => 'Streaming' + ); + + // Get the TV show title + $this->showTitle = trim($html->find('span[id=news-title]', 0)->plaintext); + + $list = $html->find('div[class=prez_7]'); + foreach($list as $element) { + $add = false; + // Link type is needed is needed to generate an unique link + $type = $this->findLinkType($element); + if($filter == 'both') { + $add = true; + } else { + if($type == $filter) { + $add = true; + } + } + if($add == true) { + $item = array(); + + // Get the element name + $title = $element->plaintext; + + // Get thee element links + $links = $element->next_sibling()->innertext; + + $item['content'] = $links; + $item['title'] = $this->showTitle . ' ' . $title . ' - ' . $typesText[$type]; + // As RSS Bridge use the URI as GUID they need to be unique : adding a md5 hash of the title element + // should geneerate unique URI to prevent confusion for RSS readers + $item['uri'] = self::URI . $this->getInput('url') . '#' . hash('md5', $item['title']); + + $this->items[] = $item; + } + } + } + + public function getName(){ + switch($this->queriedContext) { + case 'Suivre la publication des épisodes d\'une série en cours de diffusion': + return $this->showTitle . ' - ' . self::NAME; + break; + default: + return self::NAME; + } + } + + private function findLinkType($element) + { + $return = ''; + // Walk through all elements in the reverse order until finding one with class 'presz_2' + while($element->class != 'prez_2') { + $element = $element->prev_sibling(); + } + $text = html_entity_decode($element->plaintext); + + // Regarding the text of the element, return the according link type + if(stristr($text, 'téléchargement') != false) { + $return = 'download'; + } else if(stristr($text, 'streaming') != false) { + $return = 'streaming'; + } + + return $return; + } + +} diff --git a/bridges/FB2Bridge.php b/bridges/FB2Bridge.php index 1aeb30d..fd65c9f 100644 --- a/bridges/FB2Bridge.php +++ b/bridges/FB2Bridge.php @@ -15,24 +15,18 @@ class FB2Bridge extends BridgeAbstract { ) )); - public function collectData(){ - - function extractFromDelimiters($string, $start, $end){ - if(strpos($string, $start) !== false) { - $section_retrieved = substr($string, strpos($string, $start) + strlen($start)); - $section_retrieved = substr($section_retrieved, 0, strpos($section_retrieved, $end)); - return $section_retrieved; - } + public function getIcon() { + return 'https://static.xx.fbcdn.net/rsrc.php/yo/r/iRmz9lCMBD2.ico'; + } - return false; - } + public function collectData(){ //Utility function for cleaning a Facebook link $unescape_fb_link = function($matches){ if(is_array($matches) && count($matches) > 1) { $link = $matches[1]; if(strpos($link, '/') === 0) - $link = self::URI . $link . '"'; + $link = self::URI . substr($link, 1); if(strpos($link, 'facebook.com/l.php?u=') !== false) $link = urldecode(extractFromDelimiters($link, 'facebook.com/l.php?u=', '&')); return ' href="' . $link . '"'; @@ -75,14 +69,14 @@ class FB2Bridge extends BridgeAbstract { if($this->getInput('u') !== null) { $page = 'https://touch.facebook.com/' . $this->getInput('u'); $cookies = $this->getCookies($page); - $pageID = $this->getPageID($page, $cookies); + $pageInfo = $this->getPageInfos($page, $cookies); - if($pageID === null) { + if($pageInfo['userId'] === null) { echo <<<EOD Unable to get the page id. You should consider getting the ID by hand, then importing it into FB2Bridge EOD; die(); - } elseif($pageID == -1) { + } elseif($pageInfo['userId'] == -1) { echo <<<EOD This page is not accessible without being logged in. EOD; @@ -91,24 +85,31 @@ EOD; } //Build the string for the first request - $requestString = 'https://touch.facebook.com/pages_reaction_units/more/?page_id=' - . $pageID - . '&cursor={"card_id"%3A"videos"%2C"has_next_page"%3Atrue}&surface=mobile_page_home&unit_count=8'; - + $requestString = 'https://touch.facebook.com/page_content_list_view/more/?page_id=' + . $pageInfo['userId'] + . '&start_cursor=1&num_to_fetch=105&surface_type=timeline'; $fileContent = getContents($requestString); - - $articleIndex = 0; - $maxArticle = 3; - $html = $this->buildContent($fileContent); - $author = $this->getInput('u'); + $author = $pageInfo['username']; foreach($html->find('article') as $content) { $item = array(); - - $item['uri'] = 'http://touch.facebook.com' - . $content->find("div[class='_52jc _5qc4 _24u0 _36xo']", 0)->find('a', 0)->getAttribute('href'); + //echo $content; die(); + preg_match('/publish_time\\\":([0-9]+),/', $content->getAttribute('data-store', 0), $match); + if(isset($match[1])) + $timestamp = $match[1]; + else + $timestamp = 0; + + $item['uri'] = html_entity_decode('http://touch.facebook.com' + . $content->find("div[class='_52jc _5qc4 _24u0 _36xo']", 0)->find('a', 0)->getAttribute('href'), ENT_QUOTES); + + //Decode images + $imagecleaned = preg_replace_callback('/<i [^>]* style="[^"]*url\(\'(.*?)\'\).*?><\/i>/m', function ($matches) { + return "<img src='" . str_replace(['\\3a ', '\\3d ', '\\26 '], [':', '=', '&'], $matches[1]) . "' />"; + }, $content); + $content = str_get_html($imagecleaned); if($content->find('header', 0) !== null) { $content->find('header', 0)->innertext = ''; @@ -118,8 +119,13 @@ EOD; $content->find('footer', 0)->innertext = ''; } + // Replace emoticon images by their textual representation (part of the span) + foreach($content->find('span[title*="emoticon"]') as $emoticon) { + $emoticon->innertext = $emoticon->find('span[aria-hidden="true"]', 0)->innertext; + } + //Remove html nodes, keep only img, links, basic formatting - $content = strip_tags($content, '<a><img><i><u><br><p>'); + $content = strip_tags($content, '<a><img><i><u><br><p><h3><h4><section>'); //Adapt link hrefs: convert relative links into absolute links and bypass external link redirection $content = preg_replace_callback('/ href=\"([^"]+)\"/i', $unescape_fb_link, $content); @@ -132,7 +138,6 @@ EOD; 'ajaxify', 'tabindex', 'class', - 'style', 'data-[^=]*', 'aria-[^=]*', 'role', @@ -145,7 +150,36 @@ EOD; // "<i><u>smile emoticon</u></i>" back to ASCII emoticons eg ":)" $content = preg_replace_callback('/<i><u>([^ <>]+) ([^<>]+)<\/u><\/i>/i', $unescape_fb_emote, $content); - $item['content'] = $content; + //Remove the "...Plus" tag + $content = preg_replace( + '/… (<span>|)<a href="https:\/\/www\.facebook\.com\/story\.php\?story_fbid=.*?<\/a>/m', + '', $content, 1); + + //Remove tracking images + $content = preg_replace('/<img src=\'.*?safe_image\.php.*?\' \/>/m', '', $content); + + //Remove the double section tags + $content = str_replace(['<section><section>', '</section></section>'], ['<section>', '</section>'], $content); + + //Move the section tag link upper, if it is down + $content = str_get_html($content); + $sectionContent = $content->find('section', 0); + if($sectionContent != null) { + $sectionLink = $sectionContent->nextSibling(); + if($sectionLink != null) { + $fullLink = '<a href="' . $sectionLink->getAttribute('href') . '">' . $sectionContent->innertext . '</a>'; + $sectionContent->innertext = $fullLink; + } + } + + //Move the href tag upper if it is inside the section + foreach($content->find('section > a') as $sectionToFix) { + $sectionLink = $sectionToFix->getAttribute('href'); + $section = $sectionToFix->parent(); + $section->outertext = '<a href="' . $sectionLink . '">' . $section . '</a>'; + } + + $item['content'] = html_entity_decode($content, ENT_QUOTES); $title = $author; if (strlen($title) > 24) @@ -154,57 +188,29 @@ EOD; if (strlen($title) > 64) $title = substr($title, 0, strpos(wordwrap($title, 64), "\n")) . '...'; - $item['title'] = $title; - $item['author'] = $author; + $item['title'] = html_entity_decode($title, ENT_QUOTES); + $item['author'] = html_entity_decode($author, ENT_QUOTES); + $item['timestamp'] = html_entity_decode($timestamp, ENT_QUOTES); - array_push($this->items, $item); + if($item['timestamp'] != 0) + array_push($this->items, $item); } - } - - - // Currently not used. Is used to get more than only 3 elements, as they appear on another page. - private function computeNextLink($string, $pageID){ - $regex = implode( - '', - array( - '/timeline_unit', - "\\\\\\\\u00253A1", - "\\\\\\\\u00253A([0-9]*)", - "\\\\\\\\u00253A([0-9]*)", - "\\\\\\\\u00253A([0-9]*)", - "\\\\\\\\u00253A([0-9]*)/" - ) - ); - - preg_match($regex, $string, $result); - - return implode( - '', - array( - 'https://touch.facebook.com/pages_reaction_units/more/?page_id=', - $pageID, - '&cursor=%7B%22timeline_cursor%22%3A%22timeline_unit%3A1%3A', - $result[1], - '%3A', - $result[2], - '%3A', - $result[3], - '%3A', - $result[4], - '%22%2C%22timeline_section_cursor%22%3A%7B%7D%2C%22', - 'has_next_page%22%3Atrue%7D&surface=mobile_page_home&unit_count=3' - ) - ); } + //Builds the HTML from the encoded JS that Facebook provides. private function buildContent($pageContent){ // The html ends with: // /div>","replaceifexists $regex = '/\\"html\\":(\".+\/div>"),"replace/'; preg_match($regex, $pageContent, $result); - return str_get_html(html_entity_decode(json_decode($result[1]))); + + $htmlContent = json_decode($result[1]); + $htmlContent = preg_replace('/(?<!style)="(.*?)"/', '=\'$1\'', $htmlContent); + $htmlContent = html_entity_decode($htmlContent, ENT_QUOTES, 'UTF-8'); + + return str_get_html($htmlContent); } @@ -234,8 +240,8 @@ EOD; return substr($cookies, 1); } - //Get the page ID from the Facebook page. - private function getPageID($page, $cookies){ + //Get the page ID and username from the Facebook page. + private function getPageInfos($page, $cookies){ $context = stream_context_create(array( 'http' => array( @@ -251,19 +257,28 @@ EOD; return -1; } + //Get the username + $usernameRegex = '/data-nt=\"FB:TEXT4\">(.*?)<\/div>/m'; + preg_match($usernameRegex, $pageContent, $usernameMatches); + if(count($usernameMatches) > 0) { + $username = strip_tags($usernameMatches[1]); + } else { + $username = $this->getInput('u'); + } + //Get the page ID if we don't have a captcha $regex = '/page_id=([0-9]*)&/'; preg_match($regex, $pageContent, $matches); if(count($matches) > 0) { - return $matches[1]; + return array('userId' => $matches[1], 'username' => $username); } //Get the page ID if we do have a captcha $regex = '/"pageID":"([0-9]*)"/'; preg_match($regex, $pageContent, $matches); - return $matches[1]; + return array('userId' => $matches[1], 'username' => $username); } @@ -275,7 +290,4 @@ EOD; return 'http://facebook.com'; } - public function getCacheDuration(){ - return 60 * 60 * 3; // 5 minutes - } } diff --git a/bridges/FDroidBridge.php b/bridges/FDroidBridge.php index a1a37ef..b606cec 100644 --- a/bridges/FDroidBridge.php +++ b/bridges/FDroidBridge.php @@ -19,6 +19,10 @@ class FDroidBridge extends BridgeAbstract { ) )); + public function getIcon() { + return self::URI . 'assets/favicon.ico?v=8j6PKzW9Mk'; + } + public function collectData(){ $url = self::URI; $html = getSimpleHTMLDOM($url) @@ -45,9 +49,9 @@ class FDroidBridge extends BridgeAbstract { $item['icon'] = $element->find('img', 0)->src; $item['summary'] = $element->find('span.package-summary', 0)->plaintext; $item['content'] = ' - <a href="'.$item['uri'].'"> - <img alt="" style="max-height:128px" src="'.$item['icon'].'"> - </a><br>'.$item['summary']; + <a href="' . $item['uri'] . '"> + <img alt="" style="max-height:128px" src="' . $item['icon'] . '"> + </a><br>' . $item['summary']; $this->items[] = $item; } } diff --git a/bridges/FacebookBridge.php b/bridges/FacebookBridge.php index c8ea0d7..ab39c10 100644 --- a/bridges/FacebookBridge.php +++ b/bridges/FacebookBridge.php @@ -2,7 +2,7 @@ class FacebookBridge extends BridgeAbstract { const MAINTAINER = 'teromene, logmanoriginal'; - const NAME = 'Facebook'; + const NAME = 'Facebook Bridge'; const URI = 'https://www.facebook.com/'; const CACHE_TIMEOUT = 300; // 5min const DESCRIPTION = 'Input a page title or a profile log. For a profile log, @@ -41,23 +41,75 @@ class FacebookBridge extends BridgeAbstract { 'exampleValue' => 'https://www.facebook.com/groups/743149642484225', 'title' => 'Insert group name or facebook group URL' ) + ), + 'global' => array( + 'limit' => array( + 'name' => 'Limit', + 'type' => 'number', + 'required' => false, + 'title' => 'Specify the number of items to return (default: -1)', + 'defaultValue' => -1 + ) ) ); private $authorName = ''; private $groupName = ''; + public function getIcon() { + return 'https://static.xx.fbcdn.net/rsrc.php/yo/r/iRmz9lCMBD2.ico'; + } + + public function getName(){ + + switch($this->queriedContext) { + + case 'User': + if(!empty($this->authorName)) { + return isset($this->extraInfos['name']) ? $this->extraInfos['name'] : $this->authorName + . ' - ' . static::NAME; + } + break; + + case 'Group': + if(!empty($this->groupName)) { + return $this->groupName . ' - ' . static::NAME; + } + break; + + } + + return parent::getName(); + } + public function getURI() { $uri = self::URI; switch($this->queriedContext) { case 'Group': + // Discover groups via https://www.facebook.com/groups/ + // Example group: https://www.facebook.com/groups/sailors.worldwide $uri .= 'groups/' . $this->sanitizeGroup(filter_var($this->getInput('g'), FILTER_SANITIZE_URL)); break; + case 'User': + // Example user 1: https://www.facebook.com/artetv/ + // Example user 2: artetv + $user = $this->sanitizeUser($this->getInput('u')); + + if(!strpos($user, '/')) { + $uri .= urlencode($user) . '/posts'; + } else { + $uri .= 'pages/' . $user; + } + + break; + } + // Request the mobile version to reduce page size (no javascript) + // More information: https://stackoverflow.com/a/11103592 return $uri .= '?_fb_noscript=1'; } @@ -78,6 +130,12 @@ class FacebookBridge extends BridgeAbstract { } + $limit = $this->getInput('limit') ?: -1; + + if($limit > 0 && count($this->items) > $limit) { + $this->items = array_slice($this->items, 0, $limit); + } + } #region Group @@ -249,173 +307,224 @@ class FacebookBridge extends BridgeAbstract { } - #endregion + #endregion (Group) - private function collectUserData(){ + #region User - //Extract a string using start and end delimiters - function extractFromDelimiters($string, $start, $end){ - if(strpos($string, $start) !== false) { - $section_retrieved = substr($string, strpos($string, $start) + strlen($start)); - $section_retrieved = substr($section_retrieved, 0, strpos($section_retrieved, $end)); - return $section_retrieved; + /** + * Checks if $user is a valid username or URI and returns the username + */ + private function sanitizeUser($user) { + if (filter_var($user, FILTER_VALIDATE_URL)) { + + $urlparts = parse_url($user); + + if($urlparts['host'] !== parse_url(self::URI)['host']) { + returnClientError('The host you provided is invalid! Received "' + . $urlparts['host'] + . '", expected "' + . parse_url(self::URI)['host'] + . '"!'); + } + + if(!array_key_exists('path', $urlparts) + || $urlparts['path'] === '/') { + returnClientError('The URL you provided doesn\'t contain the user name!'); + } + + return explode('/', $urlparts['path'])[1]; + + } else { + + // First character cannot be a forward slash + if(strpos($user, '/') === 0) { + returnClientError('Remove leading slash "/" from the username!'); } - return false; + return $user; + } + } - //Utility function for cleaning a Facebook link - $unescape_fb_link = function($matches){ + /** + * Bypass external link redirection + */ + private function unescape_fb_link($content){ + return preg_replace_callback('/ href=\"([^"]+)\"/i', function($matches){ if(is_array($matches) && count($matches) > 1) { + $link = $matches[1]; - if(strpos($link, '/') === 0) - $link = self::URI . $link; + if(strpos($link, 'facebook.com/l.php?u=') !== false) $link = urldecode(extractFromDelimiters($link, 'facebook.com/l.php?u=', '&')); + + return ' href="' . $link . '"'; + + } + }, $content); + } + + /** + * Remove Facebook's tracking code + */ + private function remove_tracking_codes($content){ + return preg_replace_callback('/ href=\"([^"]+)\"/i', function($matches){ + if(is_array($matches) && count($matches) > 1) { + + $link = $matches[1]; + + if(strpos($link, 'facebook.com') !== false) { + if(strpos($link, '?') !== false) { + $link = substr($link, 0, strpos($link, '?')); + } + } return ' href="' . $link . '"'; + } - }; - - //Utility function for converting facebook emoticons - $unescape_fb_emote = function($matches){ - static $facebook_emoticons = array( - 'smile' => ':)', - 'frown' => ':(', - 'tongue' => ':P', - 'grin' => ':D', - 'gasp' => ':O', - 'wink' => ';)', - 'pacman' => ':<', - 'grumpy' => '>_<', - 'unsure' => ':/', - 'cry' => ':\'(', - 'kiki' => '^_^', - 'glasses' => '8-)', - 'sunglasses' => 'B-)', - 'heart' => '<3', - 'devil' => ']:D', - 'angel' => '0:)', - 'squint' => '-_-', - 'confused' => 'o_O', - 'upset' => 'xD', - 'colonthree' => ':3', - 'like' => '👍'); - $len = count($matches); - if ($len > 1) - for ($i = 1; $i < $len; $i++) - foreach ($facebook_emoticons as $name => $emote) - if ($matches[$i] === $name) - return $emote; - return $matches[0]; - }; - - $html = null; - - //Handle captcha response sent by the viewer + }, $content); + } + + /** + * Convert textual representation of emoticons back to ASCII emoticons. + * i.e. "<i><u>smile emoticon</u></i>" => ":)" + */ + private function unescape_fb_emote($content){ + return preg_replace_callback('/<i><u>([^ <>]+) ([^<>]+)<\/u><\/i>/i', function($matches){ + static $facebook_emoticons = array( + 'smile' => ':)', + 'frown' => ':(', + 'tongue' => ':P', + 'grin' => ':D', + 'gasp' => ':O', + 'wink' => ';)', + 'pacman' => ':<', + 'grumpy' => '>_<', + 'unsure' => ':/', + 'cry' => ':\'(', + 'kiki' => '^_^', + 'glasses' => '8-)', + 'sunglasses' => 'B-)', + 'heart' => '<3', + 'devil' => ']:D', + 'angel' => '0:)', + 'squint' => '-_-', + 'confused' => 'o_O', + 'upset' => 'xD', + 'colonthree' => ':3', + 'like' => '👍'); + + $len = count($matches); + + if ($len > 1) + for ($i = 1; $i < $len; $i++) + foreach ($facebook_emoticons as $name => $emote) + if ($matches[$i] === $name) + return $emote; + + return $matches[0]; + }, $content); + } + + /** + * Returns the captcha message for the given captcha + */ + private function returnCaptchaMessage($captcha) { + // Save form for submitting after getting captcha response + if (session_status() == PHP_SESSION_NONE) { + session_start(); + } + + $captcha_fields = array(); + + foreach ($captcha->find('input, button') as $input) { + $captcha_fields[$input->name] = $input->value; + } + + $_SESSION['captcha_fields'] = $captcha_fields; + $_SESSION['captcha_action'] = $captcha->find('form', 0)->action; + + // Show captcha filling form to the viewer, proxying the captcha image + $img = base64_encode(getContents($captcha->find('img', 0)->src)); + + header('Content-Type: text/html', true, 500); + + $message = <<<EOD +<form method="post" action="?{$_SERVER['QUERY_STRING']}"> +<h2>Facebook captcha challenge</h2> +<p>Unfortunately, rss-bridge cannot fetch the requested page.<br /> +Facebook wants rss-bridge to resolve the following captcha:</p> +<p><img src="data:image/png;base64,{$img}" /></p> +<p><b>Response:</b> <input name="captcha_response" placeholder="please fill in" /> +<input type="submit" value="Submit!" /></p> +</form> +EOD; + + die($message); + } + + /** + * Checks if a capture response was received and tries to load the contents + * @return mixed null if no capture response was received, simplhtmldom document otherwise + */ + private function handleCaptchaResponse() { if (isset($_POST['captcha_response'])) { if (session_status() == PHP_SESSION_NONE) session_start(); + if (isset($_SESSION['captcha_fields'], $_SESSION['captcha_action'])) { $captcha_action = $_SESSION['captcha_action']; $captcha_fields = $_SESSION['captcha_fields']; $captcha_fields['captcha_response'] = preg_replace('/[^a-zA-Z0-9]+/', '', $_POST['captcha_response']); - $header = array("Content-type: -application/x-www-form-urlencoded\r\nReferer: $captcha_action\r\nCookie: noscript=1\r\n"); + $header = array( + 'Content-type: application/x-www-form-urlencoded', + 'Referer: ' . $captcha_action, + 'Cookie: noscript=1' + ); + $opts = array( CURLOPT_POST => 1, CURLOPT_POSTFIELDS => http_build_query($captcha_fields) ); - $html = getContents($captcha_action, $header, $opts); + $html = getSimpleHTMLDOM($captcha_action, $header, $opts) + or returnServerError('Failed to submit captcha response back to Facebook'); - if($html === false) { - returnServerError('Failed to submit captcha response back to Facebook'); - } - unset($_SESSION['captcha_fields']); - $html = str_get_html($html); + return $html; } + unset($_SESSION['captcha_fields']); unset($_SESSION['captcha_action']); } - //Retrieve page contents - if(is_null($html)) { - $header = array('Accept-Language: ' . getEnv('HTTP_ACCEPT_LANGUAGE') . "\r\n"); - - // Check if the user provided a fully qualified URL - if (filter_var($this->getInput('u'), FILTER_VALIDATE_URL)) { - - $urlparts = parse_url($this->getInput('u')); - - if($urlparts['host'] !== parse_url(self::URI)['host']) { - returnClientError('The host you provided is invalid! Received "' - . $urlparts['host'] - . '", expected "' - . parse_url(self::URI)['host'] - . '"!'); - } - - if(!array_key_exists('path', $urlparts) - || $urlparts['path'] === '/') { - returnClientError('The URL you provided doesn\'t contain the user name!'); - } + return null; + } - $user = explode('/', $urlparts['path'])[1]; + private function collectUserData(){ - $html = getSimpleHTMLDOM(self::URI . urlencode($user) . '?_fb_noscript=1', $header) - or returnServerError('No results for this query.'); + $html = $this->handleCaptchaResponse(); - } else { + // Retrieve page contents + if(is_null($html)) { - // First character cannot be a forward slash - if(strpos($this->getInput('u'), '/') === 0) { - returnClientError('Remove leading slash "/" from the username!'); - } + $header = array('Accept-Language: ' . getEnv('HTTP_ACCEPT_LANGUAGE')); - if(!strpos($this->getInput('u'), '/')) { - $html = getSimpleHTMLDOM(self::URI . urlencode($this->getInput('u')) . '?_fb_noscript=1', $header) - or returnServerError('No results for this query.'); - } else { - $html = getSimpleHTMLDOM(self::URI . 'pages/' . $this->getInput('u') . '?_fb_noscript=1', $header) - or returnServerError('No results for this query.'); - } + $html = getSimpleHTMLDOM($this->getURI(), $header) + or returnServerError('No results for this query.'); - } } - //Handle captcha form? + // Handle captcha form? $captcha = $html->find('div.captcha_interstitial', 0); + if (!is_null($captcha)) { - //Save form for submitting after getting captcha response - if (session_status() == PHP_SESSION_NONE) - session_start(); - $captcha_fields = array(); - foreach ($captcha->find('input, button') as $input) - $captcha_fields[$input->name] = $input->value; - $_SESSION['captcha_fields'] = $captcha_fields; - $_SESSION['captcha_action'] = $captcha->find('form', 0)->action; - - //Show captcha filling form to the viewer, proxying the captcha image - $img = base64_encode(getContents($captcha->find('img', 0)->src)); - http_response_code(500); - header('Content-Type: text/html'); - $message = <<<EOD -<form method="post" action="?{$_SERVER['QUERY_STRING']}"> - <h2>Facebook captcha challenge</h2> - <p>Unfortunately, rss-bridge cannot fetch the requested page.<br /> - Facebook wants rss-bridge to resolve the following captcha:</p> - <p><img src="data:image/png;base64,{$img}" /></p> - <p><b>Response:</b> <input name="captcha_response" placeholder="please fill in" /> - <input type="submit" value="Submit!" /></p> -</form> -EOD; - die($message); + $this->returnCaptchaMessage($captcha); } - //No captcha? We can carry on retrieving page contents :) - //First, we check wether the page is public or not + // No captcha? We can carry on retrieving page contents :) + // First, we check wether the page is public or not $loginForm = $html->find('._585r', 0); + if($loginForm != null) { returnServerError('You must be logged in to view this page. This is not supported by RSS-Bridge.'); } @@ -424,16 +533,14 @@ EOD; ->find('#pagelet_timeline_main_column')[0] ->children(0) ->children(0) - ->children(0) ->next_sibling() ->children(0); if(isset($element)) { - $author = str_replace(' | Facebook', '', $html->find('title#pageTitle', 0)->innertext); - $profilePic = 'https://graph.facebook.com/' - . $this->getInput('u') - . '/picture?width=200&height=200'; + $author = str_replace(' - Posts | Facebook', '', $html->find('title#pageTitle', 0)->innertext); + + $profilePic = $html->find('meta[property="og:image"]', 0)->content; $this->authorName = $author; @@ -468,34 +575,53 @@ EOD; if(count($post->find('abbr')) > 0) { - //Retrieve post contents - $content = preg_replace( - '/(?i)><div class=\"clearfix([^>]+)>(.+?)div\ class=\"userContent\"/i', - '', - $post); + $content = $post->find('.userContentWrapper', 0); + + // This array specifies filters applied to all posts in order of appearance + $content_filters = array( + '._5mly', // Remove embedded videos (the preview image remains) + '._2ezg', // Remove "Views ..." + '.hidden_elem', // Remove hidden elements (they are hidden anyway) + ); + + foreach($content_filters as $filter) { + foreach($content->find($filter) as $subject) { + $subject->outertext = ''; + } + } + + // Change origin tag for embedded media from div to paragraph + foreach($content->find('._59tj') as $subject) { + $subject->outertext = '<p>' . $subject->innertext . '</p>'; + } + + // Change title tag for embedded media from anchor to paragraph + foreach($content->find('._3n1k a') as $anchor) { + $anchor->outertext = '<p>' . $anchor->innertext . '</p>'; + } $content = preg_replace( - '/(?i)><div class=\"_59tj([^>]+)>(.+?)<\/div><\/div><a/i', + '/(?i)><div class=\"_3dp([^>]+)>(.+?)div\ class=\"[^u]+userContent\"/i', '', $content); $content = preg_replace( - '/(?i)><div class=\"_3dp([^>]+)>(.+?)div\ class=\"[^u]+userContent\"/i', + '/(?i)><div class=\"_4l5([^>]+)>(.+?)<\/div>/i', '', $content); + // Remove "SpSonsSoriSsés" $content = preg_replace( - '/(?i)><div class=\"_4l5([^>]+)>(.+?)<\/div>/i', + '/(?iU)<a [^>]+ href="#" role="link" [^>}]+>.+<\/a>/iU', '', $content); - //Remove html nodes, keep only img, links, basic formatting + // Remove html nodes, keep only img, links, basic formatting $content = strip_tags($content, '<a><img><i><u><br><p>'); - //Adapt link hrefs: convert relative links into absolute links and bypass external link redirection - $content = preg_replace_callback('/ href=\"([^"]+)\"/i', $unescape_fb_link, $content); + $content = $this->unescape_fb_link($content); - //Clean useless html tag properties and fix link closing tags + // Clean useless html tag properties and fix link closing tags foreach (array( 'onmouseover', 'onclick', @@ -508,35 +634,47 @@ EOD; 'aria-[^=]*', 'role', 'rel', - 'id') as $property_name) - $content = preg_replace('/ ' . $property_name . '=\"[^"]*\"/i', '', $content); + 'id') as $property_name) { + $content = preg_replace('/ ' . $property_name . '=\"[^"]*\"/i', '', $content); + } + $content = preg_replace('/<\/a [^>]+>/i', '</a>', $content); - //Convert textual representation of emoticons eg - //"<i><u>smile emoticon</u></i>" back to ASCII emoticons eg ":)" - $content = preg_replace_callback( - '/<i><u>([^ <>]+) ([^<>]+)<\/u><\/i>/i', - $unescape_fb_emote, - $content - ); + $this->unescape_fb_emote($content); - //Retrieve date of the post + // Restore links in the post before further parsing + $post = defaultLinkTo($post, self::URI); + + // Restore links in the content before adding to the item + $content = defaultLinkTo($content, self::URI); + + $content = $this->remove_tracking_codes($content); + + // Retrieve date of the post $date = $post->find('abbr')[0]; + if(isset($date) && $date->hasAttribute('data-utime')) { $date = $date->getAttribute('data-utime'); } else { $date = 0; } - //Build title from username and content + // Build title from username and content $title = $author; + if(strlen($title) > 24) $title = substr($title, 0, strpos(wordwrap($title, 24), "\n")) . '...'; + $title = $title . ' | ' . strip_tags($content); + if(strlen($title) > 64) $title = substr($title, 0, strpos(wordwrap($title, 64), "\n")) . '...'; - $uri = self::URI . $post->find('abbr')[0]->parent()->getAttribute('href'); + $uri = $post->find('abbr')[0]->parent()->getAttribute('href'); + + if (false !== strpos($uri, '?')) { + $uri = substr($uri, 0, strpos($uri, '?')); + } //Build and add final item $item['uri'] = htmlspecialchars_decode($uri); @@ -544,6 +682,11 @@ EOD; $item['title'] = $title; $item['author'] = $author; $item['timestamp'] = $date; + + if(strpos($item['content'], '<img') === false) { + $item['enclosures'] = array($profilePic); + } + $this->items[] = $item; } } @@ -551,25 +694,6 @@ EOD; } } - public function getName(){ - - switch($this->queriedContext) { - - case 'User': - if(!empty($this->authorName)) { - return isset($this->extraInfos['name']) ? $this->extraInfos['name'] : $this->authorName - . ' - Facebook Bridge'; - } - break; - - case 'Group': - if(!empty($this->groupName)) { - return $this->groupName . ' - Facebook Bridge'; - } - break; - - } + #endregion (User) - return parent::getName(); - } } diff --git a/bridges/FierPandaBridge.php b/bridges/FierPandaBridge.php index cd9d11b..75a02cf 100644 --- a/bridges/FierPandaBridge.php +++ b/bridges/FierPandaBridge.php @@ -7,18 +7,27 @@ class FierPandaBridge extends BridgeAbstract { const CACHE_TIMEOUT = 21600; // 6h const DESCRIPTION = 'Returns latest articles from Fier Panda.'; + public function getIcon() { + return self::URI . 'wp-content/themes/fier-panda/img/favicon.png'; + } + public function collectData(){ + $html = getSimpleHTMLDOM(self::URI) or returnServerError('Could not request Fier Panda.'); - foreach($html->find('div.container-content article') as $element) { + defaultLinkTo($html, static::URI); + + foreach($html->find('article') as $article) { + $item = array(); - $item['uri'] = $this->getURI() . $element->find('a', 0)->href; - $item['title'] = trim($element->find('h1 a', 0)->innertext); - // Remove the link at the end of the article - $element->find('p a', 0)->outertext = ''; - $item['content'] = $element->find('p', 0)->innertext; + + $item['uri'] = $article->find('a', 0)->href; + $item['title'] = $article->find('a', 0)->title; + $this->items[] = $item; + } + } } diff --git a/bridges/FilterBridge.php b/bridges/FilterBridge.php index e8b451c..c644911 100644 --- a/bridges/FilterBridge.php +++ b/bridges/FilterBridge.php @@ -6,6 +6,7 @@ class FilterBridge extends FeedExpander { const NAME = 'Filter'; const CACHE_TIMEOUT = 3600; // 1h const DESCRIPTION = 'Filters a feed of your choice'; + const URI = 'https://github.com/rss-bridge/rss-bridge'; const PARAMETERS = array(array( 'url' => array( @@ -26,11 +27,34 @@ class FilterBridge extends FeedExpander { ), 'defaultValue' => 'permit', ), + 'title_from_content' => array( + 'name' => 'Generate title from content', + 'type' => 'checkbox', + 'required' => false, + ) )); protected function parseItem($newItem){ $item = parent::parseItem($newItem); + if($this->getInput('title_from_content') && array_key_exists('content', $item)) { + + $content = str_get_html($item['content']); + + $pos = strpos($item['content'], ' ', 50); + + $item['title'] = substr( + $content->plaintext, + 0, + $pos + ); + + if(strlen($content->plaintext) >= $pos) { + $item['title'] .= '...'; + } + + } + switch(true) { case $this->getFilterType() === 'permit': if (preg_match($this->getFilter(), $item['title'])) { diff --git a/bridges/FindACrewBridge.php b/bridges/FindACrewBridge.php new file mode 100644 index 0000000..c245c84 --- /dev/null +++ b/bridges/FindACrewBridge.php @@ -0,0 +1,82 @@ +<?php +class FindACrewBridge extends BridgeAbstract { + const MAINTAINER = 'couraudt'; + const NAME = 'Find A Crew Bridge'; + const URI = 'https://www.findacrew.net'; + const DESCRIPTION = 'Returns the newest sailing offers.'; + const PARAMETERS = array( + array( + 'type' => array( + 'name' => 'Type of search', + 'title' => 'Choose between finding a boat or a crew', + 'type' => 'list', + 'values' => array( + 'Find a boat' => 'boat', + 'Find a crew' => 'crew' + ) + ), + 'long' => array( + 'name' => 'Longitude of the searched location', + 'title' => 'Center the search at that longitude (e.g: -42.02)' + ), + 'lat' => array( + 'name' => 'Latitude of the searched location', + 'title' => 'Center the search at that latitude (e.g: 12.42)' + ), + 'distance' => array( + 'name' => 'Limit boundary of search in KM', + 'title' => 'Boundary of the search in kilometers when using longitude and latitude' + ) + ) + ); + + public function collectData() { + $url = $this->getURI(); + + if ($this->getInput('type') == 'boat') { + $data = array('SrhLstBtAction' => 'Create'); + } else { + $data = array('SrhLstCwAction' => 'Create'); + } + + if ($this->getInput('long') && $this->getInput('lat')) { + $data['real_LocSrh_Lng'] = $this->getInput('long'); + $data['real_LocSrh_Lat'] = $this->getInput('lat'); + if ($this->getInput('distance')) { + $data['LocDis'] = (int)$this->getInput('distance') * 1000; + } + } + + $header = array( + 'Content-Type: application/x-www-form-urlencoded' + ); + + $opts = array( + CURLOPT_CUSTOMREQUEST => 'POST', + CURLOPT_POSTFIELDS => http_build_query($data) . "\n" + ); + + $html = getSimpleHTMLDOM($url, $header, $opts) or returnClientError('No results for this query.'); + + $annonces = $html->find('.css_SrhRst'); + foreach ($annonces as $annonce) { + $item = array(); + + $img = parent::getURI() . $annonce->find('.css_LstPic img', 0)->getAttribute('src'); + $item['title'] = $annonce->find('.css_LstCtrls span', 0)->plaintext; + $item['uri'] = parent::getURI() . $annonce->find('.css_PnlCtrls a', 0)->href; + $content = $annonce->find('.css_LstDtl div', 2)->innertext; + $item['content'] = "<img src='$img' /><br>$content"; + $item['enclosures'] = array($img); + $item['categories'] = array($annonce->find('.css_AccLocCur', 0)->plaintext); + $this->items[] = $item; + } + } + + public function getURI() { + $uri = parent::getURI(); + // Those params must be in the URL + $uri .= '/en/' . $this->getInput('type') . '/search?srhtyp=srhrst&mdl=2'; + return $uri; + } +} diff --git a/bridges/FlickrBridge.php b/bridges/FlickrBridge.php index f5ebe9c..6dd3431 100644 --- a/bridges/FlickrBridge.php +++ b/bridges/FlickrBridge.php @@ -30,30 +30,76 @@ class FlickrBridge extends BridgeAbstract { 'title' => 'Insert username (as shown in the address bar)', 'exampleValue' => 'flickr' ) - ), + ) ); public function collectData(){ + switch($this->queriedContext) { + case 'Explore': - $key = 'photos'; + $filter = 'photo-lite-models'; $html = getSimpleHTMLDOM(self::URI . 'explore') or returnServerError('Could not request Flickr.'); break; + case 'By keyword': - $key = 'photos'; + $filter = 'photo-lite-models'; $html = getSimpleHTMLDOM(self::URI . 'search/?q=' . urlencode($this->getInput('q')) . '&s=rec') or returnServerError('No results for this query.'); break; + case 'By username': - $key = 'photoPageList'; + $filter = 'photo-models'; $html = getSimpleHTMLDOM(self::URI . 'photos/' . urlencode($this->getInput('u'))) or returnServerError('Requested username can\'t be found.'); break; + default: returnClientError('Invalid context: ' . $this->queriedContext); + + } + + $model_json = $this->extractJsonModel($html); + $photo_models = $this->getPhotoModels($model_json, $filter); + + foreach($photo_models as $model) { + + $item = array(); + + /* Author name depends on scope. On a keyword search the + * author is part of the picture data. On a username search + * the author is part of the owner data. + */ + if(array_key_exists('username', $model)) { + $item['author'] = $model['username']; + } elseif (array_key_exists('owner', reset($model_json)[0])) { + $item['author'] = reset($model_json)[0]['owner']['username']; + } + + $item['title'] = (array_key_exists('title', $model) ? $model['title'] : 'Untitled'); + $item['uri'] = self::URI . 'photo.gne?id=' . $model['id']; + + $description = (array_key_exists('description', $model) ? $model['description'] : ''); + + $item['content'] = '<a href="' + . $item['uri'] + . '"><img src="' + . $this->extractContentImage($model) + . '" style="max-width: 640px; max-height: 480px;"/></a><br><p>' + . $description + . '</p>'; + + $item['enclosures'] = $this->extractEnclosures($model); + + $this->items[] = $item; + } + } + + private function extractJsonModel($html) { + // Find SCRIPT containing JSON data $model = $html->find('.modelExport', 0); $model_text = $model->innertext; @@ -62,59 +108,79 @@ class FlickrBridge extends BridgeAbstract { $start = strpos($model_text, 'modelExport:') + strlen('modelExport:'); $end = strpos($model_text, 'auth:') - strlen('auth:'); - // Dissect JSON data and remove trailing comma + // Extract JSON data, remove trailing comma $model_text = trim(substr($model_text, $start, $end - $start)); $model_text = substr($model_text, 0, strlen($model_text) - 1); - $model_json = json_decode($model_text, true); + return json_decode($model_text, true); + + } + + private function getPhotoModels($json, $filter) { + + // The JSON model contains a "legend" array, where each element contains + // the path to an element in the "main" object + $photo_models = array(); + + foreach($json['legend'] as $legend) { + + $photo_model = $json['main']; + + foreach($legend as $element) { // Traverse tree + $photo_model = $photo_model[$element]; + } + + // We are only interested in content + if($photo_model['_flickrModelRegistry'] === $filter) { + $photo_models[] = $photo_model; + } + + } + + return $photo_models; - foreach($html->find('.photo-list-photo-view') as $element) { - // Get the styles - $style = explode(';', $element->style); + } - // Get the background-image style - $backgroundImage = explode(':', end($style)); + private function extractEnclosures($model) { - // URI type : url(//cX.staticflickr.com/X/XXXXX/XXXXXXXXX.jpg) - $imageURI = trim(str_replace(['url(', ')'], '', end($backgroundImage))); + $areas = array(); - // Get the image ID - $imageURIs = explode('_', basename($imageURI)); - $imageID = reset($imageURIs); + foreach($model['sizes'] as $size) { + $areas[$size['width'] * $size['height']] = $size['url']; + } - // Use JSON data to build items - foreach(reset($model_json)[0][$key]['_data'] as $element) { - if($element['id'] === $imageID) { - $item = array(); + return array($this->fixURL(max($areas))); - /* Author name depends on scope. On a keyword search the - * author is part of the picture data. On a username search - * the author is part of the owner data. - */ - if(array_key_exists('username', $element)) { - $item['author'] = $element['username']; - } elseif (array_key_exists('owner', reset($model_json)[0])) { - $item['author'] = reset($model_json)[0]['owner']['username']; - } + } - $item['title'] = (array_key_exists('title', $element) ? $element['title'] : 'Untitled'); - $item['uri'] = self::URI . 'photo.gne?id=' . $imageID; + private function extractContentImage($model) { - $description = (array_key_exists('description', $element) ? $element['description'] : ''); + $areas = array(); + $limit = 320 * 240; - $item['content'] = '<a href="' - . $item['uri'] - . '"><img src="' - . $imageURI - . '" /></a><br><p>' - . $description - . '</p>'; + foreach($model['sizes'] as $size) { - $this->items[] = $item; + $image_area = $size['width'] * $size['height']; - break; - } + if($image_area >= $limit) { + $areas[$image_area] = $size['url']; } + + } + + return $this->fixURL(min($areas)); + + } + + private function fixURL($url) { + + // For some reason the image URLs don't include the protocol (https) + if(strpos($url, '//') === 0) { + $url = 'https:' . $url; } + + return $url; + } + } diff --git a/bridges/ForGifsBridge.php b/bridges/ForGifsBridge.php new file mode 100644 index 0000000..9e76017 --- /dev/null +++ b/bridges/ForGifsBridge.php @@ -0,0 +1,41 @@ +<?php +class ForGifsBridge extends FeedExpander { + + const MAINTAINER = 'logmanoriginal'; + const NAME = 'forgifs Bridge'; + const URI = 'https://forgifs.com'; + const DESCRIPTION = 'Returns the forgifs feed with actual gifs instead of images'; + + public function collectData() { + $this->collectExpandableDatas('https://forgifs.com/gallery/srss/7'); + } + + protected function parseItem($feedItem) { + + $item = parent::parseItem($feedItem); + + $content = str_get_html($item['content']); + $img = $content->find('img', 0); + $poster = $img->src; + + // The actual gif is the same path but its id must be decremented by one. + // Example: + // http://forgifs.com/gallery/d/279419-2/Reporter-videobombed-shoulder-checks.gif + // http://forgifs.com/gallery/d/279418-2/Reporter-videobombed-shoulder-checks.gif + // Notice how this changes ----------^ + // Now let's extract that number and do some math + // Notice: Technically we could also load the content page but that would + // require unnecessary traffic. As long as it works... + $num = substr($img->src, 29, 6); + $num -= 1; + $img->src = substr_replace($img->src, $num, 29, strlen($num)); + $img->width = 'auto'; + $img->height = 'auto'; + + $item['content'] = $content; + + return $item; + + } + +} diff --git a/bridges/FourchanBridge.php b/bridges/FourchanBridge.php index ecd9d90..b20b9c1 100644 --- a/bridges/FourchanBridge.php +++ b/bridges/FourchanBridge.php @@ -69,7 +69,7 @@ class FourchanBridge extends BridgeAbstract { . '" src="' . $item['imageThumb'] . '" /></a><br>' - .$item['content']; + . $item['content']; } $this->items[] = $item; } diff --git a/bridges/FuturaSciencesBridge.php b/bridges/FuturaSciencesBridge.php index b9479c3..772f443 100644 --- a/bridges/FuturaSciencesBridge.php +++ b/bridges/FuturaSciencesBridge.php @@ -3,7 +3,7 @@ class FuturaSciencesBridge extends FeedExpander { const MAINTAINER = 'ORelio'; const NAME = 'Futura-Sciences Bridge'; - const URI = 'http://www.futura-sciences.com/'; + const URI = 'https://www.futura-sciences.com/'; const DESCRIPTION = 'Returns the newest articles.'; const PARAMETERS = array( array( @@ -90,42 +90,11 @@ class FuturaSciencesBridge extends FeedExpander { or returnServerError('Could not request Futura-Sciences: ' . $item['uri']); $item['content'] = $this->extractArticleContent($article); $author = $this->extractAuthor($article); - $item['author'] = empty($author) ? $item['author'] : $author; + if (!empty($author)) + $item['author'] = $author; return $item; } - private function stripWithDelimiters($string, $start, $end){ - while(strpos($string, $start) !== false) { - $section_to_remove = substr($string, strpos($string, $start)); - $section_to_remove = substr($section_to_remove, 0, strpos($section_to_remove, $end) + strlen($end)); - $string = str_replace($section_to_remove, '', $string); - } return $string; - } - - private function stripRecursiveHTMLSection($string, $tag_name, $tag_start){ - $open_tag = '<' . $tag_name; - $close_tag = '</' . $tag_name . '>'; - $close_tag_length = strlen($close_tag); - if(strpos($tag_start, $open_tag) === 0) { - while(strpos($string, $tag_start) !== false) { - $max_recursion = 100; - $section_to_remove = null; - $section_start = strpos($string, $tag_start); - $search_offset = $section_start; - do { - $max_recursion--; - $section_end = strpos($string, $close_tag, $search_offset); - $search_offset = $section_end + $close_tag_length; - $section_to_remove = substr($string, $section_start, $section_end - $section_start + $close_tag_length); - $open_tag_count = substr_count($section_to_remove, $open_tag); - $close_tag_count = substr_count($section_to_remove, $close_tag); - } while ($open_tag_count > $close_tag_count && $max_recursion > 0); - $string = str_replace($section_to_remove, '', $string); - } - } - return $string; - } - private function extractArticleContent($article){ $contents = $article->find('section.article-text-classic', 0)->innertext; $headline = trim($article->find('p.description', 0)->plaintext); @@ -137,6 +106,7 @@ class FuturaSciencesBridge extends FeedExpander { '<div class="sharebar2', '<div class="diaporamafullscreen"', '<div class="module social-button', + '<div class="module social-share', '<div style="margin-bottom:10px;" class="noprint"', '<div class="ficheprevnext', '<div class="bar noprint', @@ -148,16 +118,17 @@ class FuturaSciencesBridge extends FeedExpander { '<div id="forumcomments', '<div ng-if="active"' ) as $div_start) { - $contents = $this->stripRecursiveHTMLSection($contents, 'div', $div_start); + $contents = stripRecursiveHTMLSection($contents, 'div', $div_start); } - $contents = $this->stripWithDelimiters($contents, '<hr ', '/>'); - $contents = $this->stripWithDelimiters($contents, '<p class="content-date', '</p>'); - $contents = $this->stripWithDelimiters($contents, '<h1 class="content-title', '</h1>'); - $contents = $this->stripWithDelimiters($contents, 'fs:definition="', '"'); - $contents = $this->stripWithDelimiters($contents, 'fs:xt:clicktype="', '"'); - $contents = $this->stripWithDelimiters($contents, 'fs:xt:clickname="', '"'); - $contents = $this->stripWithDelimiters($contents, '<script ', '</script>'); + $contents = stripWithDelimiters($contents, '<hr ', '/>'); + $contents = stripWithDelimiters($contents, '<p class="content-date', '</p>'); + $contents = stripWithDelimiters($contents, '<h1 class="content-title', '</h1>'); + $contents = stripWithDelimiters($contents, 'fs:definition="', '"'); + $contents = stripWithDelimiters($contents, 'fs:xt:clicktype="', '"'); + $contents = stripWithDelimiters($contents, 'fs:xt:clickname="', '"'); + $contents = StripWithDelimiters($contents, '<section class="module-toretain module-propal-nl', '</section>'); + $contents = stripWithDelimiters($contents, '<script ', '</script>'); return $headline . trim($contents); } diff --git a/bridges/GBAtempBridge.php b/bridges/GBAtempBridge.php index f80a25c..9383be7 100644 --- a/bridges/GBAtempBridge.php +++ b/bridges/GBAtempBridge.php @@ -20,50 +20,58 @@ class GBAtempBridge extends BridgeAbstract { ) )); - private function extractFromDelimiters($string, $start, $end){ - if(strpos($string, $start) !== false) { - $section_retrieved = substr($string, strpos($string, $start) + strlen($start)); - $section_retrieved = substr($section_retrieved, 0, strpos($section_retrieved, $end)); - return $section_retrieved; - } - - return false; - } - - private function stripWithDelimiters($string, $start, $end){ - while(strpos($string, $start) !== false) { - $section_to_remove = substr($string, strpos($string, $start)); - $section_to_remove = substr($section_to_remove, 0, strpos($section_to_remove, $end) + strlen($end)); - $string = str_replace($section_to_remove, '', $string); - } - - return $string; - } - - private function buildItem($uri, $title, $author, $timestamp, $content){ + private function buildItem($uri, $title, $author, $timestamp, $thumbnail, $content){ $item = array(); $item['uri'] = $uri; $item['title'] = $title; $item['author'] = $author; $item['timestamp'] = $timestamp; $item['content'] = $content; + if (!empty($thumbnail)) { + $item['enclosures'] = array($thumbnail); + } return $item; } private function cleanupPostContent($content, $site_url){ $content = str_replace(':arrow:', '➤', $content); - $content = str_replace('href="attachments/', 'href="'.$site_url.'attachments/', $content); - $content = $this->stripWithDelimiters($content, '<script', '</script>'); + $content = str_replace('href="attachments/', 'href="' . $site_url . 'attachments/', $content); + $content = stripWithDelimiters($content, '<script', '</script>'); return $content; } + private function findItemDate($item){ + $time = 0; + $dateField = $item->find('abbr.DateTime', 0); + if (is_object($dateField)) { + $time = intval( + extractFromDelimiters( + $dateField->outertext, + 'data-time="', + '"' + ) + ); + } else { + $dateField = $item->find('span.DateTime', 0); + $time = DateTime::createFromFormat( + 'M j, Y \a\t g:i A', + extractFromDelimiters( + $dateField->outertext, + 'title="', + '"' + ) + )->getTimestamp(); + } + return $time; + } + private function fetchPostContent($uri, $site_url){ - $html = getSimpleHTMLDOM($uri); + $html = getSimpleHTMLDOMCached($uri); if(!$html) { - return 'Could not request GBAtemp ' . $uri; + return 'Could not request GBAtemp: ' . $uri; } - $content = $html->find('div.messageContent', 0)->innertext; + $content = $html->find('div.messageContent, blockquote.baseHtml', 0)->innertext; return $this->cleanupPostContent($content, $site_url); } @@ -76,70 +84,56 @@ class GBAtempBridge extends BridgeAbstract { case 'N': foreach($html->find('li[class=news_item full]') as $newsItem) { $url = self::URI . $newsItem->find('a', 0)->href; - $time = intval( - $this->extractFromDelimiters( - $newsItem->find('abbr.DateTime', 0)->outertext, - 'data-time="', - '"' - ) - ); + $img = $this->getURI() . $newsItem->find('img', 0)->src . '#.image'; + $time = $this->findItemDate($newsItem); $author = $newsItem->find('a.username', 0)->plaintext; $title = $newsItem->find('a', 1)->plaintext; $content = $this->fetchPostContent($url, self::URI); - $this->items[] = $this->buildItem($url, $title, $author, $time, $content); + $this->items[] = $this->buildItem($url, $title, $author, $time, $img, $content); + unset($newsItem); // Some items are heavy, freeing the item proactively helps saving memory } + break; case 'R': foreach($html->find('li.portal_review') as $reviewItem) { $url = self::URI . $reviewItem->find('a', 0)->href; + $img = $this->getURI() . extractFromDelimiters($reviewItem->find('a', 0)->style, 'image:url(', ')'); $title = $reviewItem->find('span.review_title', 0)->plaintext; $content = getSimpleHTMLDOM($url) or returnServerError('Could not request GBAtemp: ' . $uri); $author = $content->find('a.username', 0)->plaintext; - $time = intval( - $this->extractFromDelimiters( - $content->find('abbr.DateTime', 0)->outertext, - 'data-time="', - '"' - ) - ); + $time = $this->findItemDate($content); $intro = '<p><b>' . ($content->find('div#review_intro', 0)->plaintext) . '</b></p>'; $review = $content->find('div#review_main', 0)->innertext; $subheader = '<p><b>' . $content->find('div.review_subheader', 0)->plaintext . '</b></p>'; $procons = $content->find('table.review_procons', 0)->outertext; $scores = $content->find('table.reviewscores', 0)->outertext; $content = $this->cleanupPostContent($intro . $review . $subheader . $procons . $scores, self::URI); - $this->items[] = $this->buildItem($url, $title, $author, $time, $content); + $this->items[] = $this->buildItem($url, $title, $author, $time, $img, $content); + unset($reviewItem); // Free up memory } + break; case 'T': foreach($html->find('li.portal-tutorial') as $tutorialItem) { $url = self::URI . $tutorialItem->find('a', 0)->href; $title = $tutorialItem->find('a', 0)->plaintext; - $time = intval( - $this->extractFromDelimiters( - $tutorialItem->find('abbr.DateTime', 0)->outertext, - 'data-time="', - '"' - ) - ); + $time = $this->findItemDate($tutorialItem); $author = $tutorialItem->find('a.username', 0)->plaintext; $content = $this->fetchPostContent($url, self::URI); - $this->items[] = $this->buildItem($url, $title, $author, $time, $content); + $this->items[] = $this->buildItem($url, $title, $author, $time, null, $content); + unset($tutorialItem); // Free up memory } + break; case 'F': foreach($html->find('li.rc_item') as $postItem) { $url = self::URI . $postItem->find('a', 1)->href; $title = $postItem->find('a', 1)->plaintext; - $time = intval( - $this->extractFromDelimiters( - $postItem->find('abbr.DateTime', 0)->outertext, - 'data-time="', - '"' - ) - ); + $time = $this->findItemDate($postItem); $author = $postItem->find('a.username', 0)->plaintext; $content = $this->fetchPostContent($url, self::URI); - $this->items[] = $this->buildItem($url, $title, $author, $time, $content); + $this->items[] = $this->buildItem($url, $title, $author, $time, null, $content); + unset($postItem); // Free up memory } + break; } } diff --git a/bridges/GOGBridge.php b/bridges/GOGBridge.php new file mode 100644 index 0000000..79047c6 --- /dev/null +++ b/bridges/GOGBridge.php @@ -0,0 +1,66 @@ +<?php +class GOGBridge extends BridgeAbstract { + + const NAME = 'GOGBridge'; + const MAINTAINER = 'teromene'; + const URI = 'https://gog.com'; + const DESCRIPTION = 'Returns the latest releases from GOG.com'; + + public function collectData() { + + $values = getContents('https://www.gog.com/games/ajax/filtered?limit=25&sort=new') or + die('Unable to get the news pages from GOG !'); + $decodedValues = json_decode($values); + + $limit = 0; + foreach($decodedValues->products as $game) { + + $item = array(); + $item['author'] = $game->developer . ' / ' . $game->publisher; + $item['title'] = $game->title; + $item['id'] = $game->id; + $item['uri'] = self::URI . $game->url; + $item['content'] = $this->buildGameContentPage($game); + $item['timestamp'] = $game->globalReleaseDate; + + foreach($game->gallery as $image) { + $item['enclosures'][] = $image . '.jpg'; + } + + $this->items[] = $item; + $limit += 1; + + if($limit == 10) break; + + } + + } + + private function buildGameContentPage($game) { + + $gameDescriptionText = getContents('https://api.gog.com/products/' . $game->id . '?expand=description') or + die('Unable to get game description from GOG !'); + + $gameDescriptionValue = json_decode($gameDescriptionText); + + $content = 'Genres: '; + $content .= implode(', ', $game->genres); + + $content .= '<br />Supported Platforms: '; + if($game->worksOn->Windows) { + $content .= 'Windows '; + } + if($game->worksOn->Mac) { + $content .= 'Mac '; + } + if($game->worksOn->Linux) { + $content .= 'Linux '; + } + + $content .= '<br />' . $gameDescriptionValue->description->full; + + return $content; + + } + +} diff --git a/bridges/GQMagazineBridge.php b/bridges/GQMagazineBridge.php new file mode 100644 index 0000000..fa36c4e --- /dev/null +++ b/bridges/GQMagazineBridge.php @@ -0,0 +1,119 @@ +<?php + +/** + * An extension of the previous SexactuBridge to cover the whole GQMagazine. + * This one taks a page (as an example sexe/news or journaliste/maia-mazaurette) which is to be configured, + * reads all the articles visible on that page, and make a stream out of it. + * @author nicolas-delsaux + * + */ +class GQMagazineBridge extends BridgeAbstract +{ + + const MAINTAINER = 'Riduidel'; + + const NAME = 'GQMagazine'; + + // URI is no more valid, since we can address the whole gq galaxy + const URI = 'https://www.gqmagazine.fr'; + + const CACHE_TIMEOUT = 7200; // 2h + const DESCRIPTION = 'GQMagazine section extractor bridge. This bridge allows you get only a specific section.'; + + const PARAMETERS = array( array( + 'domain' => array( + 'name' => 'Domain to use', + 'required' => true, + 'values' => array( + 'www.gqmagazine.fr' => 'www.gqmagazine.fr' + ), + 'defaultValue' => 'www.gqmagazine.fr' + ), + 'page' => array( + 'name' => 'Initial page to load', + 'required' => true + ), + )); + + const REPLACED_ATTRIBUTES = array( + 'href' => 'href', + 'src' => 'src', + 'data-original' => 'src' + ); + + private function getDomain() { + return $this->getInput('domain'); + } + + public function getURI() + { + return $this->getDomain() . '/' . $this->getInput('page'); + } + + public function collectData() + { + $html = getSimpleHTMLDOM($this->getURI()) or returnServerError('Could not request ' . $this->getURI()); + + // Since GQ don't want simple class scrapping, let's do it the hard way and ... discover content ! + $main = $html->find('main', 0); + foreach ($main->find('a') as $link) { + $uri = $link->href; + $title = $link->find('h2', 0); + $date = $link->find('time', 0); + + $item = array(); + $author = $link->find('span[itemprop=name]', 0); + $item['author'] = $author->plaintext; + $item['title'] = $title->plaintext; + if(substr($uri, 0, 1) === 'h') { // absolute uri + $item['uri'] = $uri; + } else if(substr($uri, 0, 1) === '/') { // domain relative url + $item['uri'] = $this->getDomain() . $uri; + } else { + $item['uri'] = $this->getDomain() . '/' . $uri; + } + + $article = $this->loadFullArticle($item['uri']); + if($article) { + $item['content'] = $this->replaceUriInHtmlElement($article); + } else { + $item['content'] = "<strong>Article body couldn't be loaded</strong>. It must be a bug!"; + } + $short_date = $date->datetime; + $item['timestamp'] = strtotime($short_date); + $this->items[] = $item; + } + } + + /** + * Loads the full article and returns the contents + * @param $uri The article URI + * @return The article content + */ + private function loadFullArticle($uri){ + $html = getSimpleHTMLDOMCached($uri); + // Once again, that generated css classes madness is an obstacle ... which i can go over easily + foreach($html->find('div') as $div) { + // List the CSS classes of that div + $classes = $div->class; + // I can't directly lookup that class since GQ since to generate random names like "ArticleBodySection-fkggUW" + if(strpos($classes, 'ArticleBodySection') !== false) { + return $div; + } + } + return null; + } + + /** + * Replaces all relative URIs with absolute ones + * @param $element A simplehtmldom element + * @return The $element->innertext with all URIs replaced + */ + private function replaceUriInHtmlElement($element){ + $returned = $element->innertext; + foreach (self::REPLACED_ATTRIBUTES as $initial => $final) { + $returned = str_replace($initial . '="/', $final . '="' . self::URI . '/', $returned); + } + return $returned; + } +} diff --git a/bridges/GitHubGistBridge.php b/bridges/GitHubGistBridge.php new file mode 100644 index 0000000..98e4080 --- /dev/null +++ b/bridges/GitHubGistBridge.php @@ -0,0 +1,164 @@ +<?php + +class GitHubGistBridge extends BridgeAbstract { + + const NAME = 'GitHubGist comment bridge'; + const URI = 'https://gist.github.com'; + const DESCRIPTION = 'Generates feeds for Gist comments'; + const MAINTAINER = 'logmanoriginal'; + const CACHE_TIMEOUT = 3600; + + const PARAMETERS = array(array( + 'id' => array( + 'name' => 'Gist', + 'type' => 'text', + 'required' => true, + 'title' => 'Insert Gist ID or URI', + 'exampleValue' => '2646763, https://gist.github.com/2646763' + ) + )); + + private $filename; + + public function getURI() { + + $id = $this->getInput('id') ?: ''; + + $urlpath = parse_url($id, PHP_URL_PATH); + + if($urlpath) { + + $components = explode('/', $urlpath); + $id = end($components); + + } + + return static::URI . '/' . $id; + + } + + public function getName() { + return $this->filename ? $this->filename . ' - ' . static::NAME : static::NAME; + } + + public function collectData() { + + $html = getSimpleHTMLDOM($this->getURI(), + null, + null, + true, + true, + DEFAULT_TARGET_CHARSET, + false, // Do NOT remove line breaks + DEFAULT_BR_TEXT, + DEFAULT_SPAN_TEXT) + or returnServerError('Could not request ' . $this->getURI()); + + $html = defaultLinkTo($html, $this->getURI()); + + $fileinfo = $html->find('[class="file-info"]', 0) + or returnServerError('Could not find file info!'); + + $this->filename = $fileinfo->plaintext; + + $comments = $html->find('div[class="timeline-comment-wrapper"]'); + + if(is_null($comments)) { // no comments yet + return; + } + + foreach($comments as $comment) { + + $uri = $comment->find('a[href*=#gistcomment]', 0) + or returnServerError('Could not find comment anchor!'); + + $title = $comment->find('div[class="unminimized-comment"] h3[class="timeline-comment-header-text"]', 0) + or returnServerError('Could not find comment header text!'); + + $datetime = $comment->find('[datetime]', 0) + or returnServerError('Could not find comment datetime!'); + + $author = $comment->find('a.author', 0) + or returnServerError('Could not find author name!'); + + $message = $comment->find('[class="comment-body"]', 0) + or returnServerError('Could not find comment body!'); + + $item = array(); + + $item['uri'] = $uri->href; + $item['title'] = str_replace('commented', 'commented on', $title->plaintext); + $item['timestamp'] = strtotime($datetime->datetime); + $item['author'] = '<a href="' . $author->href . '">' . $author->plaintext . '</a>'; + $item['content'] = $this->fixContent($message); + // $item['enclosures'] = array(); + // $item['categories'] = array(); + + $this->items[] = $item; + + } + + } + + /** Removes all unnecessary tags and adds formatting */ + private function fixContent($content){ + + // Restore code (inside <pre />) highlighting + foreach($content->find('pre') as $pre) { + + $pre->style = <<<EOD +padding: 16px; +overflow: auto; +font-size: 85%; +line-height: 1.45; +background-color: #f6f8fa; +border-radius: 3px; +word-wrap: normal; +box-sizing: border-box; +margin-bottom: 16px; +EOD; + + $code = $pre->find('code', 0); + + if($code) { + + $code->style = <<<EOD +white-space: pre; +word-break: normal; +EOD; + + } + + } + + // find <code /> not inside <pre /> (`inline-code`) + foreach($content->find('code') as $code) { + + if($code->parent()->tag === 'pre') { + continue; + } + + $code->style = <<<EOD +background-color: rgba(27,31,35,0.05); +padding: 0.2em 0.4em; +border-radius: 3px; +EOD; + + } + + // restore text spacing + foreach($content->find('p') as $p) { + $p->style = 'margin-bottom: 16px;'; + } + + // Remove unnecessary tags + $content = strip_tags( + $content->innertext, + '<p><a><img><ol><ul><li><table><tr><th><td><string><pre><code><br><hr><h>' + ); + + return $content; + + } + +} diff --git a/bridges/GithubSearchBridge.php b/bridges/GithubSearchBridge.php index d3a615b..fe8a721 100644 --- a/bridges/GithubSearchBridge.php +++ b/bridges/GithubSearchBridge.php @@ -34,13 +34,29 @@ class GithubSearchBridge extends BridgeAbstract { $title = $element->find('h3', 0)->plaintext; $item['title'] = $title; - if (count($element->find('p')) == 2) { - $content = $element->find('p', 0)->innertext; + // Description + if (count($element->find('p.d-inline-block')) != 0) { + $content = $element->find('p.d-inline-block', 0)->innertext; } else{ - $content = ''; + $content = 'No description'; + } + + // Tags + $content = $content . '<br />'; + $tags = $element->find('a.topic-tag'); + $tags_array = array(); + if (count($tags) != 0) { + $content = $content . 'Tags : '; + foreach($tags as $tag_element) { + $tag_link = 'https://github.com' . $tag_element->href; + $tag_name = trim($tag_element->innertext); + $content = $content . '<a href="' . $tag_link . '">' . $tag_name . '</a> '; + array_push($tags_array, $tag_element->innertext); + } } - $item['content'] = $content; + $item['categories'] = $tags_array; + $item['content'] = $content; $date = $element->find('relative-time', 0)->datetime; $item['timestamp'] = strtotime($date); diff --git a/bridges/GlassdoorBridge.php b/bridges/GlassdoorBridge.php new file mode 100755 index 0000000..72b7a16 --- /dev/null +++ b/bridges/GlassdoorBridge.php @@ -0,0 +1,222 @@ +<?php +class GlassdoorBridge extends BridgeAbstract { + + // Contexts + const CONTEXT_BLOG = 'Blogs'; + const CONTEXT_REVIEW = 'Company Reviews'; + const CONTEXT_GLOBAL = 'global'; + + // Global context parameters + const PARAM_LIMIT = 'limit'; + + // Blog context parameters + const PARAM_BLOG_TYPE = 'blog_type'; + const PARAM_BLOG_FULL = 'full_article'; + + const BLOG_TYPE_HOME = 'Home'; + const BLOG_TYPE_COMPANIES_HIRING = 'Companies Hiring'; + const BLOG_TYPE_CAREER_ADVICE = 'Career Advice'; + const BLOG_TYPE_INTERVIEWS = 'Interviews'; + const BLOG_TYPE_GUIDE = 'Guides'; + + // Review context parameters + const PARAM_REVIEW_COMPANY = 'company'; + + const MAINTAINER = 'logmanoriginal'; + const NAME = 'Glassdoor Bridge'; + const URI = 'https://www.glassdoor.com/'; + const DESCRIPTION = 'Returns feeds for blog posts and company reviews'; + const CACHE_TIMEOUT = 86400; // 24 hours + + const PARAMETERS = array( + self::CONTEXT_BLOG => array( + self::PARAM_BLOG_TYPE => array( + 'name' => 'Blog type', + 'type' => 'list', + 'title' => 'Select the blog you want to follow', + 'values' => array( + self::BLOG_TYPE_HOME => 'blog/', + self::BLOG_TYPE_COMPANIES_HIRING => 'blog/companies-hiring/', + self::BLOG_TYPE_CAREER_ADVICE => 'blog/career-advice/', + self::BLOG_TYPE_INTERVIEWS => 'blog/interviews/', + self::BLOG_TYPE_GUIDE => 'blog/guide/' + ) + ), + self::PARAM_BLOG_FULL => array( + 'name' => 'Full article', + 'type' => 'checkbox', + 'title' => 'Enable to return the full article for each post' + ), + ), + self::CONTEXT_REVIEW => array( + self::PARAM_REVIEW_COMPANY => array( + 'name' => 'Company URL', + 'type' => 'text', + 'required' => true, + 'title' => 'Paste the company review page URL here!', + 'exampleValue' => 'https://www.glassdoor.com/Reviews/GitHub-Reviews-E671945.htm' + ) + ), + self::CONTEXT_GLOBAL => array( + self::PARAM_LIMIT => array( + 'name' => 'Limit', + 'type' => 'number', + 'defaultValue' => -1, + 'title' => 'Specifies the maximum number of items to return (default: All)' + ) + ) + ); + + private $host = self::URI; // They redirect without notice :/ + private $title = ''; + + public function getURI() { + switch($this->queriedContext) { + case self::CONTEXT_BLOG: + return self::URI . $this->getInput(self::PARAM_BLOG_TYPE); + case self::CONTEXT_REVIEW: + return $this->filterCompanyURI($this->getInput(self::PARAM_REVIEW_COMPANY)); + } + + return parent::getURI(); + } + + public function getName() { + return $this->title ? $this->title . ' - ' . self::NAME : parent::getName(); + } + + public function collectData() { + $html = getSimpleHTMLDOM($this->getURI()) + or returnServerError('Failed loading contents!'); + + $this->host = $html->find('link[rel="canonical"]', 0)->href; + + $html = defaultLinkTo($html, $this->host); + + $this->title = $html->find('meta[property="og:title"]', 0)->content; + $limit = $this->getInput(self::PARAM_LIMIT); + + switch($this->queriedContext) { + case self::CONTEXT_BLOG: + $this->collectBlogData($html, $limit); + break; + case self::CONTEXT_REVIEW: + $this->collectReviewData($html, $limit); + break; + } + } + + private function collectBlogData($html, $limit) { + $posts = $html->find('section') + or returnServerError('Unable to find blog posts!'); + + foreach($posts as $post) { + $item = array(); + + $item['uri'] = $post->find('header a', 0)->href; + $item['title'] = $post->find('header', 0)->plaintext; + $item['content'] = $post->find('div[class="excerpt-content"]', 0)->plaintext; + $item['enclosures'] = array( + $this->getFullSizeImageURI($post->find('div[class="post-thumb"]', 0)->{'data-original'}) + ); + + // optionally load full articles + if($this->getInput(self::PARAM_BLOG_FULL)) { + $full_html = getSimpleHTMLDOMCached($item['uri']) + or returnServerError('Unable to load full article!'); + + $full_html = defaultLinkTo($full_html, $this->host); + + $item['author'] = $full_html->find('a[rel="author"]', 0); + $item['content'] = $full_html->find('article', 0); + $item['timestamp'] = strtotime($full_html->find('time.updated', 0)->datetime); + $item['categories'] = $full_html->find('span[class="post_tag"]'); + } + + $this->items[] = $item; + + if($limit > 0 && count($this->items) >= $limit) + return; + } + } + + private function collectReviewData($html, $limit) { + $reviews = $html->find('#EmployerReviews li[id^="empReview]') + or returnServerError('Unable to find reviews!'); + + foreach($reviews as $review) { + $item = array(); + + $item['uri'] = $review->find('a.reviewLink', 0)->href; + $item['title'] = $review->find('[class="summary"]', 0)->plaintext; + $item['author'] = $review->find('div.author span', 0)->plaintext; + $item['timestamp'] = strtotime($review->find('time', 0)->datetime); + + $mainText = $review->find('p.mainText', 0)->plaintext; + $description = $review->find('div.prosConsAdvice', 0)->innertext; + $item['content'] = "<p>{$mainText}</p><p>{$description}</p>"; + + $this->items[] = $item; + + if($limit > 0 && count($this->items) >= $limit) + return; + } + } + + private function getFullSizeImageURI($uri) { + /* Images are scaled for display on the website. The scaling takes place + * on the host, who provides images in different sizes. + * + * For example: + * https://www.glassdoor.com/blog/app/uploads/sites/2/GettyImages-982402074-e1538092065712-390x193.jpg + * + * By removing the size information we receive the full sized image. + * + * For example: + * https://www.glassdoor.com/blog/app/uploads/sites/2/GettyImages-982402074-e1538092065712.jpg + */ + + $uri = filter_var($uri, FILTER_SANITIZE_URL); + return preg_replace('/(.*)(\-\d+x\d+)(\.jpg)/', '$1$3', $uri); + } + + private function filterCompanyURI($uri) { + /* Make sure the URI is a valid review page. Unfortunately there is no + * simple way to determine if the URI is valid, because of automagic + * redirection and strange naming conventions. + */ + if(!filter_var($uri, + FILTER_VALIDATE_URL, + FILTER_FLAG_SCHEME_REQUIRED | FILTER_FLAG_HOST_REQUIRED | FILTER_FLAG_PATH_REQUIRED)) { + returnClientError('The specified URL is invalid!'); + } + + $uri = filter_var($uri, FILTER_SANITIZE_URL); + $path = parse_url($uri, PHP_URL_PATH); + $parts = explode('/', $path); + + $allowed_strings = array( + 'de-DE' => 'Bewertungen', + 'en-AU' => 'Reviews', + 'nl-BE' => 'Reviews', + 'fr-BE' => 'Avis', + 'en-CA' => 'Reviews', + 'fr-CA' => 'Avis', + 'fr-FR' => 'Avis', + 'en-IN' => 'Reviews', + 'en-IE' => 'Reviews', + 'nl-NL' => 'Reviews', + 'de-AT' => 'Bewertungen', + 'de-CH' => 'Bewertungen', + 'fr-CH' => 'Avis', + 'en-GB' => 'Reviews', + 'en' => 'Reviews' + ); + + if(!in_array($parts[1], $allowed_strings)) { + returnClientError('Please specify a URL pointing to the companies review page!'); + } + + return $uri; + } +} diff --git a/bridges/GooglePlusPostBridge.php b/bridges/GooglePlusPostBridge.php index 0fccc96..cc3355e 100644 --- a/bridges/GooglePlusPostBridge.php +++ b/bridges/GooglePlusPostBridge.php @@ -1,12 +1,12 @@ <?php class GooglePlusPostBridge extends BridgeAbstract{ - protected $_title; - protected $_url; + private $title; + private $url; - const MAINTAINER = 'Grummfy'; + const MAINTAINER = 'Grummfy, logmanoriginal'; const NAME = 'Google Plus Post Bridge'; - const URI = 'https://plus.google.com/'; + const URI = 'https://plus.google.com'; const CACHE_TIMEOUT = 600; //10min const DESCRIPTION = 'Returns user public post (without API).'; @@ -14,10 +14,20 @@ class GooglePlusPostBridge extends BridgeAbstract{ 'username' => array( 'name' => 'username or Id', 'required' => true + ), + 'include_media' => array( + 'name' => 'Include media', + 'type' => 'checkbox', + 'title' => 'Enable to include media in the feed content' ) )); + public function getIcon() { + return 'https://ssl.gstatic.com/images/branding/product/ico/google_plus_alldp.ico'; + } + public function collectData(){ + $username = $this->getInput('username'); // Usernames start with a + if it's not an ID @@ -25,22 +35,20 @@ class GooglePlusPostBridge extends BridgeAbstract{ $username = '+' . $username; } - // get content parsed - $html = getSimpleHTMLDOMCached(self::URI . urlencode($username) . '/posts') + $html = getSimpleHTMLDOM(static::URI . '/' . urlencode($username) . '/posts') or returnServerError('No results for this query.'); - // get title, url, ... there is a lot of intresting stuff in meta - $this->_title = $html->find('meta[property=og:title]', 0)->getAttribute('content'); - $this->_url = $html->find('meta[property=og:url]', 0)->getAttribute('content'); + $html = defaultLinkTo($html, static::URI); + + $this->title = $html->find('meta[property=og:title]', 0)->getAttribute('content'); + $this->url = $html->find('meta[property=og:url]', 0)->getAttribute('content'); - // I don't even know where to start with this discusting html... foreach($html->find('div[jsname=WsjYwc]') as $post) { + $item = array(); - $item['author'] = $item['fullname'] = $post->find('div div div div a', 0)->innertext; - $item['id'] = $post->find('div div div', 0)->getAttribute('id'); - $item['avatar'] = $post->find('div img', 0)->src; - $item['uri'] = self::URI . $post->find('div div div a', 1)->href; + $item['author'] = $post->find('div div div div a', 0)->innertext; + $item['uri'] = $post->find('div div div a', 1)->href; $timestamp = $post->find('a.qXj2He span', 0); @@ -51,61 +59,151 @@ class GooglePlusPostBridge extends BridgeAbstract{ $timestamp->getAttribute('aria-label'))); } - // hashtag to treat : https://plus.google.com/explore/tag - // $hashtags = array(); - // foreach($post->find('a.d-s') as $hashtag){ - // $hashtags[trim($hashtag->plaintext)] = self::URI . $hashtag->href; - // } + $message = $post->find('div[jsname=EjRJtf]', 0); - $item['content'] = ''; - - // avatar display - $item['content'] .= '<div style="float:left; margin: 0 0.5em 0.5em 0;"><a href="' - . self::URI - . urlencode($this->getInput('username')); + // Empty messages are not supported right now + if(!$message) { + continue; + } - $item['content'] .= '"><img align="top" alt="' + $item['content'] = '<div style="float: left; padding: 0 10px 10px 0;"><a href="' + . $this->url + . '"><img align="top" alt="' . $item['author'] . '" src="' - . $item['avatar'] - . '" /></a></div>'; - - $content = $post->find('div[jsname=EjRJtf]', 0); - // extract plaintext - $item['content_simple'] = $content->plaintext; - $item['title'] = substr($item['content_simple'], 0, 72) . '...'; - - // XXX ugly but I don't have any idea how to do a better stuff, - // str_replace on link doesn't work as expected and ask too many checks - foreach($content->find('a') as $link) { - $hasHttp = strpos($link->href, 'http'); - $hasDoubleSlash = strpos($link->href, '//'); - - if((!$hasHttp && !$hasDoubleSlash) - || (false !== $hasHttp && strpos($link->href, 'http') != 0) - || (false === $hasHttp && false !== $hasDoubleSlash && $hasDoubleSlash != 0)) { - // skipp bad link, for some hashtag or other stuff - if(strpos($link->href, '/') == 0) { - $link->href = substr($link->href, 1); - } + . $post->find('div img', 0)->src + . '" /></a></div><div>' + . trim(strip_tags($message, '<a><p><div><img>')) + . '</div>'; + + // Make title at least 50 characters long, but don't add '...' if it is shorter! + if(strlen($message->plaintext) > 50) { + $end = strpos($message->plaintext, ' ', 50) ?: strlen($message->plaintext); + } else { + $end = strlen($message->plaintext); + } + + if(strlen(substr($message->plaintext, 0, $end)) === strlen($message->plaintext)) { + $item['title'] = $message->plaintext; + } else { + $item['title'] = substr($message->plaintext, 0, $end) . '...'; + } - $link->href = self::URI . $link->href; + $media = $post->find('[jsname="MTOxpb"]', 0); + + if($media) { + + $item['enclosures'] = array(); + + foreach($media->find('img') as $img) { + $item['enclosures'][] = $this->fixImage($img)->src; } + + if($this->getInput('include_media') === true && count($item['enclosures'] > 0)) { + $item['content'] .= '<div style="clear: both;"><a href="' + . $item['enclosures'][0] + . '"><img src="' + . $item['enclosures'][0] + . '" /></a></div>'; + } + } - $content = $content->innertext; - $item['content'] .= '<div style="margin-top: -1.5em">' . $content . '</div>'; - $item['content'] = trim(strip_tags($item['content'], '<a><p><div><img>')); + // Add custom parameters (only useful for JSON or Plaintext) + $item['fullname'] = $item['author']; + $item['avatar'] = $post->find('div img', 0)->src; + $item['id'] = $post->find('div div div', 0)->getAttribute('id'); + $item['content_simple'] = $message->plaintext; $this->items[] = $item; + } + } public function getName(){ - return $this->_title ?: 'Google Plus Post Bridge'; + return $this->title ?: 'Google Plus Post Bridge'; } public function getURI(){ - return $this->_url ?: parent::getURI(); + return $this->url ?: parent::getURI(); } + + private function fixImage($img) { + + // There are certain images like .gif which link to a static picture and + // get replaced dynamically via JS in the browser. If we want the "real" + // image we need to account for that. + + $urlparts = parse_url($img->src); + + if(array_key_exists('host', $urlparts)) { + + // For some reason some URIs don't contain the scheme, assume https + if(!array_key_exists('scheme', $urlparts)) { + $urlparts['scheme'] = 'https'; + } + + $pathelements = explode('/', $urlparts['path']); + + switch($urlparts['host']) { + + case 'lh3.googleusercontent.com': + + if(pathinfo(end($pathelements), PATHINFO_EXTENSION)) { + + // The second to last element of the path specifies the + // image format. The URL is still valid if we remove it. + unset($pathelements[count($pathelements) - 2]); + + } elseif(strrpos(end($pathelements), '=') !== false) { + + // Some images go throug a proxy. For those images they + // add size information after an equal sign. + // Example: '=w530-h298-n'. Again this can safely be + // removed to get the original image. + $pathelements[count($pathelements) - 1] = substr( + end($pathelements), + 0, + strrpos(end($pathelements), '=') + ); + + } + + break; + + } + + $urlparts['path'] = implode('/', $pathelements); + + } + + $img->src = $this->build_url($urlparts); + return $img; + + } + + /** + * From: https://gist.github.com/Ellrion/f51ba0d40ae1d62eeae44fd1adf7b704 + * slightly adjusted to work with PHP < 7.0 + * @param array $parts + * @return string + */ + private function build_url(array $parts) + { + + $scheme = isset($parts['scheme']) ? ($parts['scheme'] . '://') : ''; + $host = isset($parts['host']) ? $parts['host'] : ''; + $port = isset($parts['port']) ? (':' . $parts['port']) : ''; + $user = isset($parts['user']) ? $parts['user'] : ''; + $pass = isset($parts['pass']) ? (':' . $parts['pass']) : ''; + $pass = ($user || $pass) ? ($pass . '@') : ''; + $path = isset($parts['path']) ? $parts['path'] : ''; + $query = isset($parts['query']) ? ('?' . $parts['query']) : ''; + $fragment = isset($parts['fragment']) ? ('#' . $parts['fragment']) : ''; + + return implode('', [$scheme, $user, $pass, $host, $port, $path, $query, $fragment]); + + } + } diff --git a/bridges/GoogleSearchBridge.php b/bridges/GoogleSearchBridge.php index 2eb5841..c3d9561 100644 --- a/bridges/GoogleSearchBridge.php +++ b/bridges/GoogleSearchBridge.php @@ -28,7 +28,7 @@ class GoogleSearchBridge extends BridgeAbstract { $html = getSimpleHTMLDOM(self::URI . 'search?q=' . urlencode($this->getInput('q')) - .'&num=100&complete=0&tbs=qdr:y,sbd:1') + . '&num=100&complete=0&tbs=qdr:y,sbd:1') or returnServerError('No results for this query.'); $emIsRes = $html->find('div[id=ires]', 0); diff --git a/bridges/GrandComicsDatabaseBridge.php b/bridges/GrandComicsDatabaseBridge.php index 2b2fdfe..537a5b2 100644 --- a/bridges/GrandComicsDatabaseBridge.php +++ b/bridges/GrandComicsDatabaseBridge.php @@ -49,6 +49,7 @@ class GrandComicsDatabaseBridge extends BridgeAbstract { } // Build final item + $content = str_replace('href="/', 'href="' . static::URI, $content); $item = array(); $item['title'] = $seriesName . ' - ' . $key_date; $item['timestamp'] = strtotime($key_date); diff --git a/bridges/InstagramBridge.php b/bridges/InstagramBridge.php index 2539da2..b728567 100644 --- a/bridges/InstagramBridge.php +++ b/bridges/InstagramBridge.php @@ -3,7 +3,7 @@ class InstagramBridge extends BridgeAbstract { const MAINTAINER = 'pauder'; const NAME = 'Instagram Bridge'; - const URI = 'https://instagram.com/'; + const URI = 'https://www.instagram.com/'; const DESCRIPTION = 'Returns the newest images'; const PARAMETERS = array( @@ -19,6 +19,12 @@ class InstagramBridge extends BridgeAbstract { 'required' => true ) ), + array( + 'l' => array( + 'name' => 'location', + 'required' => true + ) + ), 'global' => array( 'media_type' => array( 'name' => 'Media type', @@ -38,16 +44,18 @@ class InstagramBridge extends BridgeAbstract { public function collectData(){ - if(!is_null($this->getInput('h')) && $this->getInput('media_type') == 'story') { - returnClientError('Stories are not supported for hashtags!'); + if(is_null($this->getInput('u')) && $this->getInput('media_type') == 'story') { + returnClientError('Stories are not supported for hashtags nor locations!'); } $data = $this->getInstagramJSON($this->getURI()); if(!is_null($this->getInput('u'))) { $userMedia = $data->entry_data->ProfilePage[0]->graphql->user->edge_owner_to_timeline_media->edges; - } else { + } elseif(!is_null($this->getInput('h'))) { $userMedia = $data->entry_data->TagPage[0]->graphql->hashtag->edge_hashtag_to_media->edges; + } elseif(!is_null($this->getInput('l'))) { + $userMedia = $data->entry_data->LocationsPage[0]->graphql->location->edge_location_to_media->edges; } foreach($userMedia as $media) { @@ -85,7 +93,7 @@ class InstagramBridge extends BridgeAbstract { $item['content'] = $data[0]; $item['enclosures'] = $data[1]; } else { - $item['content'] = '<img src="' . htmlentities($media->display_url) . '" alt="'. $item['title'] . '" />'; + $item['content'] = '<img src="' . htmlentities($media->display_url) . '" alt="' . $item['title'] . '" />'; $item['enclosures'] = array($media->display_url); } @@ -101,16 +109,21 @@ class InstagramBridge extends BridgeAbstract { $mediaInfo = $data->entry_data->PostPage[0]->graphql->shortcode_media; //Process the first element, that isn't in the node graph - $caption = $mediaInfo->edge_media_to_caption->edges[0]->node->text; + if (count($mediaInfo->edge_media_to_caption->edges) > 0) { + $caption = $mediaInfo->edge_media_to_caption->edges[0]->node->text; + } else { + $caption = ''; + } $enclosures = [$mediaInfo->display_url]; - $content = '<img src="' . htmlentities($mediaInfo->display_url) . '" alt="'. $caption . '" />'; + $content = '<img src="' . htmlentities($mediaInfo->display_url) . '" alt="' . $caption . '" />'; foreach($mediaInfo->edge_sidecar_to_children->edges as $media) { - - $content .= '<img src="' . htmlentities($media->node->display_url) . '" alt="'. $caption . '" />'; - $enclosures[] = $media->node->display_url; - + $display_url = $media->node->display_url; + if(!in_array($display_url, $enclosures)) { // add only if not added yet + $content .= '<img src="' . htmlentities($display_url) . '" alt="' . $caption . '" />'; + $enclosures[] = $display_url; + } } return [$content, $enclosures]; @@ -139,11 +152,12 @@ class InstagramBridge extends BridgeAbstract { public function getURI(){ if(!is_null($this->getInput('u'))) { - return self::URI . urlencode($this->getInput('u')); + return self::URI . urlencode($this->getInput('u')) . '/'; } elseif(!is_null($this->getInput('h'))) { return self::URI . 'explore/tags/' . urlencode($this->getInput('h')); + } elseif(!is_null($this->getInput('l'))) { + return self::URI . 'explore/locations/' . urlencode($this->getInput('l')); } - return parent::getURI(); } } diff --git a/bridges/InstructablesBridge.php b/bridges/InstructablesBridge.php new file mode 100644 index 0000000..05f1a91 --- /dev/null +++ b/bridges/InstructablesBridge.php @@ -0,0 +1,370 @@ +<?php +/** +* This class implements a bridge for http://www.instructables.com, supporting +* general feeds and feeds by category. Instructables doesn't support HTTPS as +* of now (23.06.2018), so all connections are insecure! +* +* Remarks: +* - For some reason it is very important to have the category URI end with a +* slash, otherwise the site defaults to the main category (i.e. Technology)! +* If you need to update the categories list, enable the 'listCategories' +* function (see comments below) and run the bridge with format=Html (see page +* source) +*/ +class InstructablesBridge extends BridgeAbstract { + const NAME = 'Instructables Bridge'; + const URI = 'http://www.instructables.com'; + const DESCRIPTION = 'Returns general feeds and feeds by category'; + const MAINTAINER = 'logmanoriginal'; + const PARAMETERS = array( + 'Category' => array( + 'category' => array( + 'name' => 'Category', + 'type' => 'list', + 'required' => true, + 'values' => array( + 'Play' => array( + 'All' => '/play/', + 'KNEX' => '/play/knex/', + 'Offbeat' => '/play/offbeat/', + 'Lego' => '/play/lego/', + 'Airsoft' => '/play/airsoft/', + 'Card Games' => '/play/card-games/', + 'Guitars' => '/play/guitars/', + 'Instruments' => '/play/instruments/', + 'Magic Tricks' => '/play/magic-tricks/', + 'Minecraft' => '/play/minecraft/', + 'Music' => '/play/music/', + 'Nerf' => '/play/nerf/', + 'Nintendo' => '/play/nintendo/', + 'Office Supplies' => '/play/office-supplies/', + 'Paintball' => '/play/paintball/', + 'Paper Airplanes' => '/play/paper-airplanes/', + 'Party Tricks' => '/play/party-tricks/', + 'PlayStation' => '/play/playstation/', + 'Pranks and Humor' => '/play/pranks-and-humor/', + 'Puzzles' => '/play/puzzles/', + 'Siege Engines' => '/play/siege-engines/', + 'Sports' => '/play/sports/', + 'Table Top' => '/play/table-top/', + 'Toys' => '/play/toys/', + 'Video Games' => '/play/video-games/', + 'Wii' => '/play/wii/', + 'Xbox' => '/play/xbox/', + 'Yo-Yo' => '/play/yo-yo/', + ), + 'Craft' => array( + 'All' => '/craft/', + 'Art' => '/craft/art/', + 'Sewing' => '/craft/sewing/', + 'Paper' => '/craft/paper/', + 'Jewelry' => '/craft/jewelry/', + 'Fashion' => '/craft/fashion/', + 'Books & Journals' => '/craft/books-and-journals/', + 'Cards' => '/craft/cards/', + 'Clay' => '/craft/clay/', + 'Duct Tape' => '/craft/duct-tape/', + 'Embroidery' => '/craft/embroidery/', + 'Felt' => '/craft/felt/', + 'Fiber Arts' => '/craft/fiber-arts/', + 'Gifts & Wrapping' => '/craft/gifts-and-wrapping/', + 'Knitting & Crocheting' => '/craft/knitting-and-crocheting/', + 'Leather' => '/craft/leather/', + 'Mason Jars' => '/craft/mason-jars/', + 'No-Sew' => '/craft/no-sew/', + 'Parties & Weddings' => '/craft/parties-and-weddings/', + 'Print Making' => '/craft/print-making/', + 'Soap' => '/craft/soap/', + 'Wallets' => '/craft/wallets/', + ), + 'Technology' => array( + 'All' => '/technology/', + 'Electronics' => '/technology/electronics/', + 'Arduino' => '/technology/arduino/', + 'Photography' => '/technology/photography/', + 'Leds' => '/technology/leds/', + 'Science' => '/technology/science/', + 'Reuse' => '/technology/reuse/', + 'Apple' => '/technology/apple/', + 'Computers' => '/technology/computers/', + '3D Printing' => '/technology/3D-Printing/', + 'Robots' => '/technology/robots/', + 'Art' => '/technology/art/', + 'Assistive Tech' => '/technology/assistive-technology/', + 'Audio' => '/technology/audio/', + 'Clocks' => '/technology/clocks/', + 'CNC' => '/technology/cnc/', + 'Digital Graphics' => '/technology/digital-graphics/', + 'Gadgets' => '/technology/gadgets/', + 'Kits' => '/technology/kits/', + 'Laptops' => '/technology/laptops/', + 'Lasers' => '/technology/lasers/', + 'Linux' => '/technology/linux/', + 'Microcontrollers' => '/technology/microcontrollers/', + 'Microsoft' => '/technology/microsoft/', + 'Mobile' => '/technology/mobile/', + 'Raspberry Pi' => '/technology/raspberry-pi/', + 'Remote Control' => '/technology/remote-control/', + 'Sensors' => '/technology/sensors/', + 'Software' => '/technology/software/', + 'Soldering' => '/technology/soldering/', + 'Speakers' => '/technology/speakers/', + 'Steampunk' => '/technology/steampunk/', + 'Tools' => '/technology/tools/', + 'USB' => '/technology/usb/', + 'Wearables' => '/technology/wearables/', + 'Websites' => '/technology/websites/', + 'Wireless' => '/technology/wireless/', + ), + 'Workshop' => array( + 'All' => '/workshop/', + 'Woodworking' => '/workshop/woodworking/', + 'Tools' => '/workshop/tools/', + 'Gardening' => '/workshop/gardening/', + 'Cars' => '/workshop/cars/', + 'Metalworking' => '/workshop/metalworking/', + 'Cardboard' => '/workshop/cardboard/', + 'Electric Vehicles' => '/workshop/electric-vehicles/', + 'Energy' => '/workshop/energy/', + 'Furniture' => '/workshop/furniture/', + 'Home Improvement' => '/workshop/home-improvement/', + 'Home Theater' => '/workshop/home-theater/', + 'Hydroponics' => '/workshop/hydroponics/', + 'Laser Cutting' => '/workshop/laser-cutting/', + 'Lighting' => '/workshop/lighting/', + 'Molds & Casting' => '/workshop/molds-and-casting/', + 'Motorcycles' => '/workshop/motorcycles/', + 'Organizing' => '/workshop/organizing/', + 'Pallets' => '/workshop/pallets/', + 'Repair' => '/workshop/repair/', + 'Shelves' => '/workshop/shelves/', + 'Solar' => '/workshop/solar/', + 'Workbenches' => '/workshop/workbenches/', + ), + 'Home' => array( + 'All' => '/home/', + 'Halloween' => '/home/halloween/', + 'Decorating' => '/home/decorating/', + 'Organizing' => '/home/organizing/', + 'Pets' => '/home/pets/', + 'Life Hacks' => '/home/life-hacks/', + 'Beauty' => '/home/beauty/', + 'Christmas' => '/home/christmas/', + 'Cleaning' => '/home/cleaning/', + 'Education' => '/home/education/', + 'Finances' => '/home/finances/', + 'Gardening' => '/home/gardening/', + 'Green' => '/home/green/', + 'Health' => '/home/health/', + 'Hiding Places' => '/home/hiding-places/', + 'Holidays' => '/home/holidays/', + 'Homesteading' => '/home/homesteading/', + 'Kids' => '/home/kids/', + 'Kitchen' => '/home/kitchen/', + 'Life Skills' => '/home/life-skills/', + 'Parenting' => '/home/parenting/', + 'Pest Control' => '/home/pest-control/', + 'Relationships' => '/home/relationships/', + 'Reuse' => '/home/reuse/', + 'Travel' => '/home/travel/', + ), + 'Outside' => array( + 'All' => '/outside/', + 'Bikes' => '/outside/bikes/', + 'Survival' => '/outside/survival/', + 'Backyard' => '/outside/backyard/', + 'Beach' => '/outside/beach/', + 'Birding' => '/outside/birding/', + 'Boats' => '/outside/boats/', + 'Camping' => '/outside/camping/', + 'Climbing' => '/outside/climbing/', + 'Fire' => '/outside/fire/', + 'Fishing' => '/outside/fishing/', + 'Hunting' => '/outside/hunting/', + 'Kites' => '/outside/kites/', + 'Knives' => '/outside/knives/', + 'Knots' => '/outside/knots/', + 'Paracord' => '/outside/paracord/', + 'Rockets' => '/outside/rockets/', + 'Skateboarding' => '/outside/skateboarding/', + 'Snow' => '/outside/snow/', + 'Water' => '/outside/water/', + ), + 'Food' => array( + 'All' => '/food/', + 'Dessert' => '/food/dessert/', + 'Snacks & Appetizers' => '/food/snacks-and-appetizers/', + 'Bacon' => '/food/bacon/', + 'BBQ & Grilling' => '/food/bbq-and-grilling/', + 'Beverages' => '/food/beverages/', + 'Bread' => '/food/bread/', + 'Breakfast' => '/food/breakfast/', + 'Cake' => '/food/cake/', + 'Candy' => '/food/candy/', + 'Canning & Preserves' => '/food/canning-and-preserves/', + 'Cocktails & Mocktails' => '/food/cocktails-and-mocktails/', + 'Coffee' => '/food/coffee/', + 'Cookies' => '/food/cookies/', + 'Cupcakes' => '/food/cupcakes/', + 'Homebrew' => '/food/homebrew/', + 'Main Course' => '/food/main-course/', + 'Pasta' => '/food/pasta/', + 'Pie' => '/food/pie/', + 'Pizza' => '/food/pizza/', + 'Salad' => '/food/salad/', + 'Sandwiches' => '/food/sandwiches/', + 'Soups & Stews' => '/food/soups-and-stews/', + 'Vegetarian & Vegan' => '/food/vegetarian-and-vegan/', + ), + 'Costumes' => array( + 'All' => '/costumes/', + 'Props' => '/costumes/props-and-accessories/', + 'Animals' => '/costumes/animals/', + 'Comics' => '/costumes/comics/', + 'Fantasy' => '/costumes/fantasy/', + 'For Kids' => '/costumes/for-kids/', + 'For Pets' => '/costumes/for-pets/', + 'Funny' => '/costumes/funny/', + 'Games' => '/costumes/games/', + 'Historic & Futuristic' => '/costumes/historic-and-futuristic/', + 'Makeup' => '/costumes/makeup/', + 'Masks' => '/costumes/masks/', + 'Scary' => '/costumes/scary/', + 'TV & Movies' => '/costumes/tv-and-movies/', + 'Weapons & Armor' => '/costumes/weapons-and-armor/', + ) + ), + 'title' => 'Select your category (required)', + 'defaultValue' => 'Technology' + ), + 'filter' => array( + 'name' => 'Filter', + 'type' => 'list', + 'required' => true, + 'values' => array( + 'Featured' => ' ', + 'Recent' => 'recent/', + 'Popular' => 'popular/', + 'Views' => 'views/', + 'Contest Winners' => 'winners/' + ), + 'title' => 'Select a filter', + 'defaultValue' => 'Featured' + ) + ) + ); + + private $uri; + + public function collectData() { + // Enable the following line to get the category list (dev mode) + // $this->listCategories(); + + $this->uri = static::URI; + + switch($this->queriedContext) { + case 'Category': $this->uri .= $this->getInput('category') . $this->getInput('filter'); + } + + $html = getSimpleHTMLDOM($this->uri) + or returnServerError('Error loading category ' . $this->uri); + + foreach($html->find('ul.explore-covers-list li') as $cover) { + $item = array(); + + $item['uri'] = static::URI . $cover->find('a.cover-image', 0)->href; + $item['title'] = $cover->find('.title', 0)->innertext; + $item['author'] = $this->getCategoryAuthor($cover); + $item['content'] = '<a href=' + . $item['uri'] + . '><img src=' + . $cover->find('a.cover-image img', 0)->src + . '></a>'; + + $image = str_replace('.RECTANGLE1', '.LARGE', $cover->find('a.cover-image img', 0)->src); + $item['enclosures'] = [$image]; + + $this->items[] = $item; + } + } + + public function getName() { + if(!is_null($this->getInput('category')) + && !is_null($this->getInput('filter'))) { + foreach(self::PARAMETERS[$this->queriedContext]['category']['values'] as $key => $value) { + $subcategory = array_search($this->getInput('category'), $value); + + if($subcategory !== false) + break; + } + + $filter = array_search( + $this->getInput('filter'), + self::PARAMETERS[$this->queriedContext]['filter']['values'] + ); + + return $subcategory . ' (' . $filter . ') - ' . static::NAME; + } + + return parent::getName(); + } + + public function getURI() { + if(!is_null($this->getInput('category')) + && !is_null($this->getInput('filter'))) { + return $this->uri; + } + + return parent::getURI(); + } + + /** + * Returns a list of categories for development purposes (used to build the + * parameters list) + */ + private function listCategories(){ + // Use arbitrary category to receive full list + $html = getSimpleHTMLDOM(self::URI . '/technology/'); + + foreach($html->find('.channel a') as $channel) { + $name = html_entity_decode(trim($channel->innertext)); + + // Remove unwanted entities + $name = str_replace("'", '', $name); + $name = str_replace(''', '', $name); + + $uri = $channel->href; + + $category = explode('/', $uri)[1]; + + if(!isset($categories) + || !array_key_exists($category, $categories) + || !in_array($uri, $categories[$category])) + $categories[$category][$name] = $uri; + } + + // Build PHP array manually + foreach($categories as $key => $value) { + $name = ucfirst($key); + echo "'{$name}' => array(\n"; + echo "\t'All' => '/{$key}/',\n"; + foreach($value as $name => $uri) { + echo "\t'{$name}' => '{$uri}',\n"; + } + echo "),\n"; + } + + die; + } + + /** + * Returns the author as anchor for a given cover. + */ + private function getCategoryAuthor($cover) { + return '<a href=' + . static::URI . $cover->find('span.author a', 0)->href + . '>' + . $cover->find('span.author a', 0)->innertext + . '</a>'; + } +} diff --git a/bridges/JapanExpoBridge.php b/bridges/JapanExpoBridge.php index c80bb24..1790171 100644 --- a/bridges/JapanExpoBridge.php +++ b/bridges/JapanExpoBridge.php @@ -3,7 +3,7 @@ class JapanExpoBridge extends BridgeAbstract { const MAINTAINER = 'Ginko'; const NAME = 'Japan Expo Actualités'; - const URI = 'http://www.japan-expo-paris.com/fr/actualites'; + const URI = 'https://www.japan-expo-paris.com/fr/actualites'; const CACHE_TIMEOUT = 14400; // 4h const DESCRIPTION = 'Returns most recent entries from Japan Expo actualités.'; const PARAMETERS = array( array( @@ -13,6 +13,10 @@ class JapanExpoBridge extends BridgeAbstract { ) )); + public function getIcon() { + return 'https://s.japan-expo.com/katana/images/JES073/favicons/paris.png'; + } + public function collectData(){ function frenchPubDateToTimestamp($date_to_parse) { @@ -51,7 +55,7 @@ class JapanExpoBridge extends BridgeAbstract { foreach($html->find('a._tile2') as $element) { $url = $element->href; - $thumbnail = 'http://s.japan-expo.com/katana/images/JES049/paris.png'; + $thumbnail = 'https://s.japan-expo.com/katana/images/JES049/paris.png'; preg_match('/url\(([^)]+)\)/', $element->find('img.rspvimgset', 0)->style, $img_search_result); if(count($img_search_result) >= 2) @@ -62,7 +66,8 @@ class JapanExpoBridge extends BridgeAbstract { break; } - $article_html = getSimpleHTMLDOMCached('Could not request JapanExpo: ' . $url); + $article_html = getSimpleHTMLDOMCached($url) + or returnServerError('Could not request JapanExpo: ' . $url); $header = $article_html->find('header.pageHeadBox', 0); $timestamp = strtotime($header->find('time', 0)->datetime); $title_html = $header->find('div.section', 0)->next_sibling(); @@ -92,6 +97,7 @@ class JapanExpoBridge extends BridgeAbstract { $item['uri'] = $url; $item['title'] = $title; $item['timestamp'] = $timestamp; + $item['enclosures'] = array($thumbnail); $item['content'] = $content; $this->items[] = $item; $count++; diff --git a/bridges/KATBridge.php b/bridges/KATBridge.php index c4325a6..55756bf 100644 --- a/bridges/KATBridge.php +++ b/bridges/KATBridge.php @@ -36,6 +36,11 @@ class KATBridge extends BridgeAbstract { 'name' => 'Only get results from Elite or Verified uploaders ?', ), )); + + public function getIcon() { + return 'https://statuskatcrco-631f.kxcdn.com/assets/images/favicon.ico'; + } + public function collectData(){ function parseDateTimestamp($element){ $guessedDate = strptime($element, '%d-%m-%Y %H:%M:%S'); diff --git a/bridges/KernelBugTrackerBridge.php b/bridges/KernelBugTrackerBridge.php index f3135af..5fdb2a5 100644 --- a/bridges/KernelBugTrackerBridge.php +++ b/bridges/KernelBugTrackerBridge.php @@ -38,6 +38,10 @@ class KernelBugTrackerBridge extends BridgeAbstract { private $bugid = ''; private $bugdesc = ''; + public function getIcon() { + return self::URI . '/images/favicon.ico'; + } + public function collectData(){ $limit = $this->getInput('limit'); $sorting = $this->getInput('sorting'); diff --git a/bridges/KununuBridge.php b/bridges/KununuBridge.php index 85f7c7b..2f4bf0b 100644 --- a/bridges/KununuBridge.php +++ b/bridges/KununuBridge.php @@ -64,7 +64,7 @@ class KununuBridge extends BridgeAbstract { return parent::getURI(); } - function getName(){ + public function getName(){ if(!is_null($this->getInput('company'))) { $company = $this->fixCompanyName($this->getInput('company')); return ($this->companyName ?: $company) . ' - ' . self::NAME; @@ -73,52 +73,67 @@ class KununuBridge extends BridgeAbstract { return parent::getName(); } + public function getIcon() { + return 'https://www.kununu.com/favicon-196x196.png'; + } + public function collectData(){ $full = $this->getInput('full'); // Load page - $html = getSimpleHTMLDOMCached($this->getURI()); - if(!$html) - returnServerError('Unable to receive data from ' . $this->getURI() . '!'); + $html = getSimpleHTMLDOM($this->getURI()) + or returnServerError('Unable to receive data from ' . $this->getURI() . '!'); + + $html = defaultLinkTo($html, static::URI); + // Update name for this request - $this->companyName = $this->extractCompanyName($html); + $company = $html->find('span[class="company-name"]', 0) + or returnServerError('Cannot find company name!'); + + $this->companyName = $company->innertext; // Find the section with all the panels (reviews) - $section = $html->find('section.kununu-scroll-element', 0); - if($section === false) - returnServerError('Unable to find panel section!'); + $section = $html->find('section.kununu-scroll-element', 0) + or returnServerError('Unable to find panel section!'); // Find all articles (within the panels) - $articles = $section->find('article'); - if($articles === false || empty($articles)) - returnServerError('Unable to find articles!'); + $articles = $section->find('article') + or returnServerError('Unable to find articles!'); // Go through all articles foreach($articles as $article) { + + $anchor = $article->find('h1.review-title a', 0) + or returnServerError('Cannot find article URI!'); + + $date = $article->find('meta[itemprop=dateCreated]', 0) + or returnServerError('Cannot find article date!'); + + $rating = $article->find('span.rating', 0) + or returnServerError('Cannot find article rating!'); + + $summary = $article->find('[itemprop=name]', 0) + or returnServerError('Cannot find article summary!'); + $item = array(); $item['author'] = $this->extractArticleAuthorPosition($article); - $item['timestamp'] = $this->extractArticleDate($article); - $item['title'] = $this->extractArticleRating($article) + $item['timestamp'] = strtotime($date); + $item['title'] = $rating->getAttribute('aria-label') . ' : ' - . $this->extractArticleSummary($article); + . strip_tags($summary->innertext); - $item['uri'] = $this->extractArticleUri($article); + $item['uri'] = $anchor->href; - if($full) + if($full) { $item['content'] = $this->extractFullDescription($item['uri']); - else + } else { $item['content'] = $this->extractArticleDescription($article); + } $this->items[] = $item; - } - } - /** - * Fixes relative URLs in the given text - */ - private function fixUrl($text){ - return preg_replace('/href=(\'|\")\//i', 'href="'.self::URI, $text); + } } /* @@ -128,73 +143,11 @@ class KununuBridge extends BridgeAbstract { $company = trim($company); $company = str_replace(' ', '-', $company); $company = strtolower($company); - return $this->encodeUmlauts($company); - } - /** - * Encodes unmlauts in the given text - */ - private function encodeUmlauts($text){ $umlauts = Array('/ä/','/ö/','/ü/','/Ä/','/Ö/','/Ü/','/ß/'); $replace = Array('ae','oe','ue','Ae','Oe','Ue','ss'); - return preg_replace($umlauts, $replace, $text); - } - - /** - * Returns the company name from the review html - */ - private function extractCompanyName($html){ - $company_name = $html->find('h1[itemprop=name]', 0); - if(is_null($company_name)) - returnServerError('Cannot find company name!'); - - return $company_name->plaintext; - } - - /** - * Returns the date from a given article - */ - private function extractArticleDate($article){ - // They conviniently provide a time attribute for us :) - $date = $article->find('meta[itemprop=dateCreated]', 0); - if(is_null($date)) - returnServerError('Cannot find article date!'); - - return strtotime($date->content); - } - - /** - * Returns the rating from a given article - */ - private function extractArticleRating($article){ - $rating = $article->find('span.rating', 0); - if(is_null($rating)) - returnServerError('Cannot find article rating!'); - - return $rating->getAttribute('aria-label'); - } - - /** - * Returns the summary from a given article - */ - private function extractArticleSummary($article){ - $summary = $article->find('[itemprop=name]', 0); - if(is_null($summary)) - returnServerError('Cannot find article summary!'); - - return strip_tags($summary->innertext); - } - - /** - * Returns the URI from a given article - */ - private function extractArticleUri($article){ - $anchor = $article->find('h1.review-title a', 0); - if(is_null($anchor)) - returnServerError('Cannot find article URI!'); - - return self::URI . $anchor->href; + return preg_replace($umlauts, $replace, $company); } /** @@ -202,9 +155,8 @@ class KununuBridge extends BridgeAbstract { */ private function extractArticleAuthorPosition($article){ // We need to parse the user-content manually - $user_content = $article->find('div.user-content', 0); - if(is_null($user_content)) - returnServerError('Cannot find user content!'); + $user_content = $article->find('div.user-content', 0) + or returnServerError('Cannot find user content!'); // Go through all h2 elements to find index of required span (I know... it's stupid) $author_position = 'Unknown'; @@ -222,11 +174,10 @@ class KununuBridge extends BridgeAbstract { * Returns the description from a given article */ private function extractArticleDescription($article){ - $description = $article->find('[itemprop=reviewBody]', 0); - if(is_null($description)) - returnServerError('Cannot find article description!'); + $description = $article->find('[itemprop=reviewBody]', 0) + or returnServerError('Cannot find article description!'); - return $this->fixUrl($description->innertext); + return $description->innertext; } /** @@ -234,14 +185,14 @@ class KununuBridge extends BridgeAbstract { */ private function extractFullDescription($uri){ // Load full article - $html = getSimpleHTMLDOMCached($uri); - if($html === false) - returnServerError('Could not load full description!'); + $html = getSimpleHTMLDOMCached($uri) + or returnServerError('Could not load full description!'); + + $html = defaultLinkTo($html, static::URI); // Find the article - $article = $html->find('article', 0); - if(is_null($article)) - returnServerError('Cannot find article!'); + $article = $html->find('article', 0) + or returnServerError('Cannot find article!'); // Luckily they use the same layout for the review overview and full article pages :) return $this->extractArticleDescription($article); diff --git a/bridges/LWNprevBridge.php b/bridges/LWNprevBridge.php index 0b79aeb..baa30c9 100644 --- a/bridges/LWNprevBridge.php +++ b/bridges/LWNprevBridge.php @@ -9,7 +9,7 @@ class LWNprevBridge extends BridgeAbstract{ private $editionTimeStamp; function getURI(){ - return self::URI.'free/bigpage'; + return self::URI . 'free/bigpage'; } private function jumpToNextTag(&$node){ @@ -47,7 +47,7 @@ class LWNprevBridge extends BridgeAbstract{ <html><head><title>LWN</title></head><body>{$content}</body></html> EOD; } else { - $content = $content.'</body></html>'; + $content = $content . '</body></html>'; } libxml_use_internal_errors(true); @@ -172,7 +172,7 @@ EOD; $prefix = ''; if(!empty($cats[0])) { - $prefix .= '['.$cats[0].($cats[1] ? '/'.$cats[1] : '').'] '; + $prefix .= '[' . $cats[0] . ($cats[1] ? '/' . $cats[1] : '') . '] '; } return $prefix; } @@ -188,7 +188,7 @@ EOD; $item = array(); - $item['uri'] = self::URI.'#'.count($items); + $item['uri'] = self::URI . '#' . count($items); $item['timestamp'] = $this->editionTimeStamp; @@ -197,7 +197,7 @@ EOD; $cat = $newsletters->previousSibling; $this->jumpToPreviousTag($cat); $prefix = $this->getItemPrefix($cat, $cats); - $item['title'] = $prefix.' '.$newsletters->textContent; + $item['title'] = $prefix . ' ' . $newsletters->textContent; $node = $newsletters; $content = ''; @@ -233,7 +233,7 @@ EOD; $cat = $cat->previousSibling; $this->jumpToPreviousTag($cat); $prefix = $this->getItemPrefix($cat, $cats); - $item['title'] = $prefix.' '.$title->textContent; + $item['title'] = $prefix . ' ' . $title->textContent; $items[] = array_merge($item, $this->getArticleContent($title)); } @@ -255,7 +255,7 @@ EOD; $cat = $cat->previousSibling; $this->jumpToPreviousTag($cat); $prefix = $this->getItemPrefix($cat, $cats); - $item['title'] = $prefix.' '.$title->textContent; + $item['title'] = $prefix . ' ' . $title->textContent; $items[] = array_merge($item, $this->getArticleContent($title)); } diff --git a/bridges/LeBonCoinBridge.php b/bridges/LeBonCoinBridge.php index 927b43e..b2cffcf 100644 --- a/bridges/LeBonCoinBridge.php +++ b/bridges/LeBonCoinBridge.php @@ -8,8 +8,8 @@ class LeBonCoinBridge extends BridgeAbstract { const PARAMETERS = array( array( - 'k' => array('name' => 'Mot Clé'), - 'r' => array( + 'keywords' => array('name' => 'Mots-Clés'), + 'region' => array( 'name' => 'Région', 'type' => 'list', 'values' => array( @@ -42,7 +42,114 @@ class LeBonCoinBridge extends BridgeAbstract { 'Réunion' => '26' ) ), - 'c' => array( + 'department' => array( + 'name' => 'Département', + 'type' => 'list', + 'values' => array( + '' => '', + 'Ain' => '1', + 'Aisne' => '2', + 'Allier' => '3', + 'Alpes-de-Haute-Provence' => '4', + 'Hautes-Alpes' => '5', + 'Alpes-Maritimes' => '6', + 'Ardèche' => '7', + 'Ardennes' => '8', + 'Ariège' => '9', + 'Aube' => '10', + 'Aude' => '11', + 'Aveyron' => '12', + 'Bouches-du-Rhône' => '13', + 'Calvados' => '14', + 'Cantal' => '15', + 'Charente' => '16', + 'Charente-Maritime' => '17', + 'Cher' => '18', + 'Corrèze' => '19', + 'Corse-du-Sud' => '2A', + 'Haute-Corse' => '2B', + 'Côte-d\'Or' => '21', + 'Côtes-d\'Armor' => '22', + 'Creuse' => '23', + 'Dordogne' => '24', + 'Doubs' => '25', + 'Drôme' => '26', + 'Eure' => '27', + 'Eure-et-Loir' => '28', + 'Finistère' => '29', + 'Gard' => '30', + 'Haute-Garonne' => '31', + 'Gers' => '32', + 'Gironde' => '33', + 'Hérault' => '34', + 'Ille-et-Vilaine' => '35', + 'Indre' => '36', + 'Indre-et-Loire' => '37', + 'Isère' => '38', + 'Jura' => '39', + 'Landes' => '40', + 'Loir-et-Cher' => '41', + 'Loire' => '42', + 'Haute-Loire' => '43', + 'Loire-Atlantique' => '44', + 'Loiret' => '45', + 'Lot' => '46', + 'Lot-et-Garonne' => '47', + 'Lozère' => '48', + 'Maine-et-Loire' => '49', + 'Manche' => '50', + 'Marne' => '51', + 'Haute-Marne' => '52', + 'Mayenne' => '53', + 'Meurthe-et-Moselle' => '54', + 'Meuse' => '55', + 'Morbihan' => '56', + 'Moselle' => '57', + 'Nièvre' => '58', + 'Nord' => '59', + 'Oise' => '60', + 'Orne' => '61', + 'Pas-de-Calais' => '62', + 'Puy-de-Dôme' => '63', + 'Pyrénées-Atlantiques' => '64', + 'Hautes-Pyrénées' => '65', + 'Pyrénées-Orientales' => '66', + 'Bas-Rhin' => '67', + 'Haut-Rhin' => '68', + 'Rhône' => '69', + 'Haute-Saône' => '70', + 'Saône-et-Loire' => '71', + 'Sarthe' => '72', + 'Savoie' => '73', + 'Haute-Savoie' => '74', + 'Paris' => '75', + 'Seine-Maritime' => '76', + 'Seine-et-Marne' => '77', + 'Yvelines' => '78', + 'Deux-Sèvres' => '79', + 'Somme' => '80', + 'Tarn' => '81', + 'Tarn-et-Garonne' => '82', + 'Var' => '83', + 'Vaucluse' => '84', + 'Vendée' => '85', + 'Vienne' => '86', + 'Haute-Vienne' => '87', + 'Vosges' => '88', + 'Yonne' => '89', + 'Territoire de Belfort' => '90', + 'Essonne' => '91', + 'Hauts-de-Seine' => '92', + 'Seine-Saint-Denis' => '93', + 'Val-de-Marne' => '94', + 'Val-d\'Oise' => '95' + ) + ), + 'cities' => array( + 'name' => 'Villes', + 'title' => 'Codes postaux séparés par des virgules' + ), + 'category' => array( 'name' => 'Catégorie', 'type' => 'list', 'values' => array( @@ -51,7 +158,7 @@ class LeBonCoinBridge extends BridgeAbstract { 'Emploi et recrutement' => '71', 'Offres d\'emploi et jobs' => '33' ), - 'VEHICULES' => array( + 'VÉHICULES' => array( 'Tous' => '1', 'Voitures' => '2', 'Motos' => '3', @@ -78,7 +185,7 @@ class LeBonCoinBridge extends BridgeAbstract { 'Hôtels' => '69', 'Hébergements insolites' => '70' ), - 'MULTIMEDIA' => array( + 'MULTIMÉDIA' => array( 'Tous' => '14', 'Informatique' => '15', 'Consoles & Jeux vidéo' => '43', @@ -98,7 +205,7 @@ class LeBonCoinBridge extends BridgeAbstract { 'Jeux & Jouets' => '41', 'Vins & Gastronomie' => '48' ), - 'MATERIEL PROFESSIONNEL' => array( + 'MATÉRIEL PROFESSIONNEL' => array( 'Tous' => '56', 'Matériel Agricole' => '57', 'Transport - Manutention' => '58', @@ -114,14 +221,14 @@ class LeBonCoinBridge extends BridgeAbstract { 'Tous' => '31', 'Prestations de services' => '34', 'Billetterie' => '35', - 'Evénements' => '49', + 'Événements' => '49', 'Cours particuliers' => '36', 'Covoiturage' => '65' ), 'MAISON' => array( 'Tous' => '18', 'Ameublement' => '19', - 'Electroménager' => '20', + 'Électroménager' => '20', 'Arts de la table' => '45', 'Décoration' => '39', 'Linge de maison' => '46', @@ -131,53 +238,145 @@ class LeBonCoinBridge extends BridgeAbstract { 'Chaussures' => '53', 'Accessoires & Bagagerie' => '47', 'Montres & Bijoux' => '42', - 'Equipement bébé' => '23', + 'Équipement bébé' => '23', 'Vêtements bébé' => '54', ), 'AUTRES' => '37' ) ), - 'o' => array( + 'pricemin' => array( + 'name' => 'Prix min', + 'type' => 'number' + ), + 'pricemax' => array( + 'name' => 'Prix max', + 'type' => 'number' + ), + 'estate' => array( + 'name' => 'Type de bien', + 'type' => 'list', + 'values' => array( + '' => '', + 'Maison' => '1', + 'Appartement' => '2', + 'Terrain' => '3', + 'Parking' => '4', + 'Autre' => '5' + ) + ), + 'roomsmin' => array( + 'name' => 'Pièces min', + 'type' => 'number' + ), + 'roomsmax' => array( + 'name' => 'Pièces max', + 'type' => 'number' + ), + 'squaremin' => array( + 'name' => 'Surface min', + 'type' => 'number' + ), + 'squaremax' => array( + 'name' => 'Surface max', + 'type' => 'number' + ), + 'mileagemin' => array( + 'name' => 'Kilométrage min', + 'type' => 'number' + ), + 'mileagemax' => array( + 'name' => 'Kilométrage max', + 'type' => 'number' + ), + 'yearmin' => array( + 'name' => 'Année min', + 'type' => 'number' + ), + 'yearmax' => array( + 'name' => 'Année max', + 'type' => 'number' + ), + 'cubiccapacitymin' => array( + 'name' => 'Cylindrée min', + 'type' => 'number' + ), + 'cubiccapacitymax' => array( + 'name' => 'Cylindrée max', + 'type' => 'number' + ), + 'fuel' => array( + 'name' => 'Énergie', + 'type' => 'list', + 'values' => array( + '' => '', + 'Essence' => '1', + 'Diesel' => '2', + 'GPL' => '3', + 'Électrique' => '4', + 'Hybride' => '6', + 'Autre' => '5' + ) + ), + 'owner' => array( 'name' => 'Vendeur', 'type' => 'list', 'values' => array( 'Tous' => '', 'Particuliers' => 'private', - 'Professionnels' => 'pro', + 'Professionnels' => 'pro' ) ) ) ); - public function collectData(){ + public static $LBC_API_KEY = 'ba0c2dad52b3ec'; - $params = array( - 'text' => $this->getInput('k'), - 'region' => $this->getInput('r'), - 'category' => $this->getInput('c'), - 'owner_type' => $this->getInput('o'), - ); + private function getRange($field, $range_min, $range_max){ - $url = self::URI . 'recherche/?' . http_build_query($params); - $html = getContents($url) - or returnServerError('Could not request LeBonCoin. Tried: ' . $url); + if(!is_null($range_min) + && !is_null($range_max) + && $range_min > $range_max) { + returnClientError('Min-' . $field . ' must be lower than max-' . $field . '.'); + } - if(!preg_match('/^<script>window.FLUX_STATE[^\r\n]*/m', $html, $matches)) { - returnServerError('Could not parse JSON in page content.'); + if(!is_null($range_min) + && is_null($range_max)) { + returnClientError('Max-' . $field . ' is needed when min-' . $field . ' is setted (range).'); } - $clean_match = str_replace( - array('</script>', '<script>window.FLUX_STATE = '), - array('', ''), - $matches[0] + return array( + 'min' => $range_min, + 'max' => $range_max + ); + } + + public function collectData(){ + + $url = 'https://api.leboncoin.fr/finder/search/'; + $data = $this->buildRequestJson(); + + $header = array( + 'Content-Type: application/json', + 'Content-Length: ' . strlen($data), + 'api_key: ' . self::$LBC_API_KEY ); - $json = json_decode($clean_match); - if($json->adSearch->data->total === 0) { + $opts = array( + CURLOPT_CUSTOMREQUEST => 'POST', + CURLOPT_POSTFIELDS => $data + + ); + + $content = getContents($url, $header, $opts) + or returnServerError('Could not request LeBonCoin. Tried: ' . $url); + + $json = json_decode($content); + + if($json->total === 0) { return; } - foreach($json->adSearch->data->ads as $element) { + foreach($json->ads as $element) { $item['title'] = $element->subject; $item['content'] = $element->body; @@ -219,4 +418,121 @@ class LeBonCoinBridge extends BridgeAbstract { $this->items[] = $item; } } + + + private function buildRequestJson() { + + $requestJson = new StdClass(); + $requestJson->owner_type = $this->getInput('owner'); + $requestJson->filters = new StdClass(); + + $requestJson->filters->keywords = array( + 'text' => $this->getInput('keywords') + ); + + if($this->getInput('region') != '') { + $requestJson->filters->location['regions'] = [$this->getInput('region')]; + } + + if($this->getInput('department') != '') { + $requestJson->filters->location['departments'] = [$this->getInput('department')]; + } + + if($this->getInput('cities') != '') { + + $requestJson->filters->location['city_zipcodes'] = array(); + + foreach (explode(',', $this->getInput('cities')) as $zipcode) { + + $requestJson->filters->location['city_zipcodes'][] = array( + 'zipcode' => trim($zipcode) + ); + } + + } + + $requestJson->filters->category = array( + 'id' => $this->getInput('category') + ); + + if($this->getInput('pricemin') != '' + || $this->getInput('pricemax') != '') { + + $requestJson->filters->ranges->price = $this->getRange( + 'price', + $this->getInput('pricemin'), + $this->getInput('pricemax') + ); + + } + + if($this->getInput('estate') != '') { + $requestJson->filters->enums['real_estate_type'] = [$this->getInput('estate')]; + } + + if($this->getInput('roomsmin') != '' + || $this->getInput('roomsmax') != '') { + + $requestJson->filters->ranges->rooms = $this->getRange( + 'rooms', + $this->getInput('roomsmin'), + $this->getInput('roomsmax') + ); + + } + + if($this->getInput('squaremin') != '' + || $this->getInput('squaremax') != '') { + + $requestJson->filters->ranges->square = $this->getRange( + 'square', + $this->getInput('squaremin'), + $this->getInput('squaremax') + ); + + } + + if($this->getInput('mileagemin') != '' + || $this->getInput('mileagemax') != '') { + + $requestJson->filters->ranges->mileage = $this->getRange( + 'mileage', + $this->getInput('mileagemin'), + $this->getInput('mileagemax') + ); + + } + + if($this->getInput('yearmin') != '' + || $this->getInput('yearmax') != '') { + + $requestJson->filters->ranges->regdate = $this->getRange( + 'year', + $this->getInput('yearmin'), + $this->getInput('yearmax') + ); + + } + + if($this->getInput('cubiccapacitymin') != '' + || $this->getInput('cubiccapacitymax') != '') { + + $requestJson->filters->ranges->cubic_capacity = $this->getRange( + 'cubic_capacity', + $this->getInput('cubiccapacitymin'), + $this->getInput('cubiccapacitymax') + ); + + } + + if($this->getInput('fuel') != '') { + $requestJson->filters->enums['fuel'] = [$this->getInput('fuel')]; + } + + $requestJson->limit = 30; + + return json_encode($requestJson); + + } + } diff --git a/bridges/LeMondeInformatiqueBridge.php b/bridges/LeMondeInformatiqueBridge.php index 706752f..09bcf6a 100644 --- a/bridges/LeMondeInformatiqueBridge.php +++ b/bridges/LeMondeInformatiqueBridge.php @@ -3,8 +3,7 @@ class LeMondeInformatiqueBridge extends FeedExpander { const MAINTAINER = 'ORelio'; const NAME = 'Le Monde Informatique'; - const URI = 'http://www.lemondeinformatique.fr/'; - const CACHE_TIMEOUT = 1800; // 30min + const URI = 'https://www.lemondeinformatique.fr/'; const DESCRIPTION = 'Returns the newest articles.'; public function collectData(){ @@ -15,30 +14,26 @@ class LeMondeInformatiqueBridge extends FeedExpander { $item = parent::parseItem($newsItem); $article_html = getSimpleHTMLDOMCached($item['uri']) or returnServerError('Could not request LeMondeInformatique: ' . $item['uri']); - $item['content'] = $this->cleanArticle($article_html->find('div#article', 0)->innertext); - $item['title'] = $article_html->find('h1.cleanprint-title', 0)->plaintext; - return $item; - } - private function stripCDATA($string){ - $string = str_replace('<![CDATA[', '', $string); - $string = str_replace(']]>', '', $string); - return $string; - } + //Deduce thumbnail URL from article image URL + $item['enclosures'] = array( + str_replace( + '/grande/', + '/petite/', + $article_html->find('.article-image', 0)->find('img', 0)->src + ) + ); - private function stripWithDelimiters($string, $start, $end){ - while(strpos($string, $start) !== false) { - $section_to_remove = substr($string, strpos($string, $start)); - $section_to_remove = substr($section_to_remove, 0, strpos($section_to_remove, $end) + strlen($end)); - $string = str_replace($section_to_remove, '', $string); - } + //No response header sets the encoding, explicit conversion is needed or subsequent xml_encode() will fail + $item['content'] = utf8_encode($this->cleanArticle($article_html->find('div.col-primary', 0)->innertext)); + $item['author'] = utf8_encode($article_html->find('div.author-infos', 0)->find('b', 0)->plaintext); - return $string; + return $item; } private function cleanArticle($article_html){ - $article_html = $this->stripWithDelimiters($article_html, '<script', '</script>'); - $article_html = $this->stripWithDelimiters($article_html, '<h1 class="cleanprint-title"', '</h1>'); + $article_html = stripWithDelimiters($article_html, '<script', '</script>'); + $article_html = explode('<p class="contact-error', $article_html)[0] . '</div>'; return $article_html; } } diff --git a/bridges/LegifranceJOBridge.php b/bridges/LegifranceJOBridge.php index d97fcff..41a9b06 100644 --- a/bridges/LegifranceJOBridge.php +++ b/bridges/LegifranceJOBridge.php @@ -38,6 +38,10 @@ class LegifranceJOBridge extends BridgeAbstract { return $item; } + public function getIcon() { + return 'https://www.legifrance.gouv.fr/img/favicon.ico'; + } + public function collectData(){ $html = getSimpleHTMLDOM(self::URI) or $this->returnServer('Unable to download ' . self::URI); diff --git a/bridges/LesJoiesDuCodeBridge.php b/bridges/LesJoiesDuCodeBridge.php index 5f61f95..0957d92 100644 --- a/bridges/LesJoiesDuCodeBridge.php +++ b/bridges/LesJoiesDuCodeBridge.php @@ -3,7 +3,7 @@ class LesJoiesDuCodeBridge extends BridgeAbstract { const MAINTAINER = 'superbaillot.net'; const NAME = 'Les Joies Du Code'; - const URI = 'http://lesjoiesducode.fr/'; + const URI = 'https://lesjoiesducode.fr/'; const CACHE_TIMEOUT = 7200; // 2h const DESCRIPTION = 'LesJoiesDuCode'; @@ -27,15 +27,7 @@ class LesJoiesDuCodeBridge extends BridgeAbstract { } $content = $temp->innertext; - $auteur = $temp->find('i', 0); - $pos = strpos($auteur->innertext, 'by'); - - if($pos > 0) { - $auteur = trim(str_replace('*/', '', substr($auteur->innertext, ($pos + 2)))); - $item['author'] = $auteur; - } - - $item['content'] .= trim($content); + $item['content'] = trim($content); $item['uri'] = $url; $item['title'] = trim($titre); diff --git a/bridges/NeuviemeArtBridge.php b/bridges/NeuviemeArtBridge.php index d0954fc..8c5bb70 100644 --- a/bridges/NeuviemeArtBridge.php +++ b/bridges/NeuviemeArtBridge.php @@ -6,16 +6,6 @@ class NeuviemeArtBridge extends FeedExpander { const URI = 'http://www.9emeart.fr/'; const DESCRIPTION = 'Returns the newest articles.'; - private function stripWithDelimiters($string, $start, $end){ - while(strpos($string, $start) !== false) { - $section_to_remove = substr($string, strpos($string, $start)); - $section_to_remove = substr($section_to_remove, 0, strpos($section_to_remove, $end) + strlen($end)); - $string = str_replace($section_to_remove, '', $string); - } - - return $string; - } - protected function parseItem($item){ $item = parent::parseItem($item); @@ -34,16 +24,16 @@ class NeuviemeArtBridge extends FeedExpander { } $article_content = ''; - if($article_image) { + if ($article_image) { $article_content = '<p><img src="' . $article_image . '" /></p>'; } $article_content .= str_replace( 'src="/', 'src="' . self::URI, $article_html->find('div.newsGenerique_con', 0)->innertext ); - $article_content = $this->stripWithDelimiters($article_content, '<script', '</script>'); - $article_content = $this->stripWithDelimiters($article_content, '<style', '</style>'); - $article_content = $this->stripWithDelimiters($article_content, '<link', '>'); + $article_content = stripWithDelimiters($article_content, '<script', '</script>'); + $article_content = stripWithDelimiters($article_content, '<style', '</style>'); + $article_content = stripWithDelimiters($article_content, '<link', '>'); $item['content'] = $article_content; diff --git a/bridges/NextInpactBridge.php b/bridges/NextInpactBridge.php index 5de5c8b..c6bf2f5 100644 --- a/bridges/NextInpactBridge.php +++ b/bridges/NextInpactBridge.php @@ -6,29 +6,105 @@ class NextInpactBridge extends FeedExpander { const URI = 'https://www.nextinpact.com/'; const DESCRIPTION = 'Returns the newest articles.'; + const PARAMETERS = array( array( + 'feed' => array( + 'name' => 'Feed', + 'type' => 'list', + 'values' => array( + 'Tous nos articles' => 'news', + 'Nos contenus en accès libre' => 'acces-libre', + 'Blog' => 'blog', + 'Bons plans' => 'bonsplans' + ) + ), + 'filter_premium' => array( + 'name' => 'Premium', + 'type' => 'list', + 'values' => array( + 'No filter' => '0', + 'Hide Premium' => '1', + 'Only Premium' => '2' + ) + ), + 'filter_brief' => array( + 'name' => 'Brief', + 'type' => 'list', + 'values' => array( + 'No filter' => '0', + 'Hide Brief' => '1', + 'Only Brief' => '2' + ) + ) + )); + public function collectData(){ - $this->collectExpandableDatas(self::URI . 'rss/news.xml', 10); + $feed = $this->getInput('feed'); + if (empty($feed)) + $feed = 'news'; + $this->collectExpandableDatas(self::URI . 'rss/' . $feed . '.xml'); } protected function parseItem($newsItem){ $item = parent::parseItem($newsItem); - $item['content'] = $this->extractContent($item['uri']); + $item['content'] = $this->extractContent($item, $item['uri']); + if (is_null($item['content'])) + return null; //Filtered article return $item; } - private function extractContent($url){ - $html2 = getSimpleHTMLDOMCached($url); - $text = '<p><em>' - . $html2->find('span.sub_title', 0)->innertext - . '</em></p><p><img src="' - . $html2->find('div.container_main_image_article', 0)->find('img.dedicated', 0)->src - . '" alt="-" /></p><div>' - . $html2->find('div[itemprop=articleBody]', 0)->innertext - . '</div>'; - - $premium_article = $html2->find('h2.title_reserve_article', 0); - if (is_object($premium_article)) - $text = $text . '<p><em>' . $premium_article->innertext . '</em></p>'; + private function extractContent($item, $url){ + $html = getSimpleHTMLDOMCached($url); + if (!is_object($html)) + return 'Failed to request NextInpact: ' . $url; + + foreach(array( + 'filter_premium' => 'h2.title_reserve_article', + 'filter_brief' => 'div.brief-inner-content' + ) as $param_name => $selector) { + $param_val = intval($this->getInput($param_name)); + if ($param_val != 0) { + $element_present = is_object($html->find($selector, 0)); + $element_wanted = ($param_val == 2); + if ($element_present != $element_wanted) { + return null; //Filter article + } + } + } + + if (is_object($html->find('div[itemprop=articleBody], div.brief-inner-content', 0))) { + + $subtitle = trim($html->find('span.sub_title, div.brief-head', 0)); + if(is_object($subtitle) && $subtitle->plaintext !== $item['title']) { + $subtitle = '<p><em>' . $subtitle->plaintext . '</em></p>'; + } else { + $subtitle = ''; + } + + $postimg = $html->find( + 'div.container_main_image_article, div.image-brief-container, div.image-brief-side-container', 0 + ); + if(is_object($postimg)) { + $postimg = '<p><img src="' + . $postimg->find('img.dedicated', 0)->src + . '" alt="-" /></p>'; + } else { + $postimg = ''; + } + + $text = $subtitle + . $postimg + . $html->find('div[itemprop=articleBody], div.brief-inner-content', 0)->outertext; + + } else { + $text = $item['content'] + . '<p><em>Failed retrieve full article content</em></p>'; + } + + $premium_article = $html->find('h2.title_reserve_article', 0); + if (is_object($premium_article)) { + $text .= '<p><em>' . $premium_article->innertext . '</em></p>'; + } + return $text; } } diff --git a/bridges/NextgovBridge.php b/bridges/NextgovBridge.php index 370b0bf..74bfc54 100644 --- a/bridges/NextgovBridge.php +++ b/bridges/NextgovBridge.php @@ -32,43 +32,39 @@ class NextgovBridge extends FeedExpander { protected function parseItem($newsItem){ $item = parent::parseItem($newsItem); - $item['content'] = ''; + $article_thumbnail = 'https://cdn.nextgov.com/nextgov/images/logo.png'; + $item['content'] = '<p><b>' . $item['content'] . '</b></p>'; $namespaces = $newsItem->getNamespaces(true); if(isset($namespaces['media'])) { $media = $newsItem->children($namespaces['media']); if(isset($media->content)) { $attributes = $media->content->attributes(); - $item['content'] = '<img src="' . $attributes['url'] . '">'; + $item['content'] = '<p><img src="' . $attributes['url'] . '"></p>' . $item['content']; + $article_thumbnail = str_replace( + 'large.jpg', + 'small.jpg', + strval($attributes['url']) + ); } } + $item['enclosures'] = array($article_thumbnail); $item['content'] .= $this->extractContent($item['uri']); return $item; } - private function stripWithDelimiters($string, $start, $end){ - while (strpos($string, $start) !== false) { - $section_to_remove = substr($string, strpos($string, $start)); - $section_to_remove = substr($section_to_remove, 0, strpos($section_to_remove, $end) + strlen($end)); - $string = str_replace($section_to_remove, '', $string); - } - - return $string; - } - private function extractContent($url){ - $article = getSimpleHTMLDOMCached($url) - or returnServerError('Could not request Nextgov: ' . $url); + $article = getSimpleHTMLDOMCached($url); + + if (!is_object($article)) + return 'Could not request Nextgov: ' . $url; - $contents = $article->find('div.wysiwyg', 0)->innertext; - $contents = $this->stripWithDelimiters($contents, '<div class="ad-container">', '</div>'); - $contents = $this->stripWithDelimiters($contents, '<div', '</div>'); //ad outer div - return $this->stripWithDelimiters($contents, '<script', '</script>'); - $contents = ($article_thumbnail == '' ? '' : '<p><img src="' . $article_thumbnail . '" /></p>') - . '<p><b>' - . $article_subtitle - . '</b></p>' - . trim($contents); + $contents = $article->find('div.wysiwyg', 0); + $contents->find('svg.content-tombstone', 0)->outertext = ''; + $contents = $contents->innertext; + $contents = stripWithDelimiters($contents, '<div class="ad-container">', '</div>'); + $contents = stripWithDelimiters($contents, '<div', '</div>'); //ad outer div + return trim(stripWithDelimiters($contents, '<script', '</script>')); } } diff --git a/bridges/NineGagBridge.php b/bridges/NineGagBridge.php new file mode 100644 index 0000000..f526135 --- /dev/null +++ b/bridges/NineGagBridge.php @@ -0,0 +1,331 @@ +<?php + +class NineGagBridge extends BridgeAbstract { + const NAME = '9gag Bridge'; + const URI = 'https://9gag.com/'; + const DESCRIPTION = 'Returns latest quotes from 9gag.'; + const MAINTAINER = 'ZeNairolf'; + const CACHE_TIMEOUT = 3600; + const PARAMETERS = array( + 'Popular' => array( + 'd' => array( + 'name' => 'Section', + 'type' => 'list', + 'required' => true, + 'values' => array( + 'Hot' => 'hot', + 'Trending' => 'trending', + 'Fresh' => 'fresh', + ), + ), + 'p' => array( + 'name' => 'Pages', + 'type' => 'number', + 'defaultValue' => 3, + ), + ), + 'Sections' => array( + 'g' => array( + 'name' => 'Section', + 'type' => 'list', + 'required' => true, + 'values' => array( + 'Animals' => 'cute', + 'Anime & Manga' => 'anime-manga', + 'Ask 9GAG' => 'ask9gag', + 'Awesome' => 'awesome', + 'Basketball' => 'basketball', + 'Car' => 'car', + 'Classical Art Memes' => 'classicalartmemes', + 'Comic' => 'comic', + 'Cosplay' => 'cosplay', + 'Countryballs' => 'country', + 'DIY & Crafts' => 'imadedis', + 'Drawing & Illustration' => 'drawing', + 'Fan Art' => 'animefanart', + 'Food & Drinks' => 'food', + 'Football' => 'football', + 'Fortnite' => 'fortnite', + 'Funny' => 'funny', + 'GIF' => 'gif', + 'Gaming' => 'gaming', + 'Girl' => 'girl', + 'Girly Things' => 'girly', + 'Guy' => 'guy', + 'History' => 'history', + 'Home Design' => 'home', + 'Horror' => 'horror', + 'K-Pop' => 'kpop', + 'LEGO' => 'lego', + 'League of Legends' => 'leagueoflegends', + 'Movie & TV' => 'movie-tv', + 'Music' => 'music', + 'NFK - Not For Kids' => 'nsfw', + 'Overwatch' => 'overwatch', + 'PC Master Race' => 'pcmr', + 'PUBG' => 'pubg', + 'Pic Of The Day' => 'photography', + 'Pokémon' => 'pokemon', + 'Politics' => 'politics', + 'Relationship' => 'relationship', + 'Roast Me' => 'roastme', + 'Satisfying' => 'satisfying', + 'Savage' => 'savage', + 'School' => 'school', + 'Sci-Tech' => 'science', + 'Sport' => 'sport', + 'Star Wars' => 'starwars', + 'Superhero' => 'superhero', + 'Surreal Memes' => 'surrealmemes', + 'Timely' => 'timely', + 'Travel' => 'travel', + 'Video' => 'video', + 'WTF' => 'wtf', + 'Wallpaper' => 'wallpaper', + 'Warhammer' => 'warhammer', + ), + ), + 't' => array( + 'name' => 'Type', + 'type' => 'list', + 'required' => true, + 'values' => array( + 'Hot' => 'hot', + 'Fresh' => 'fresh', + ), + ), + 'p' => array( + 'name' => 'Pages', + 'type' => 'number', + 'defaultValue' => 3, + ), + ), + ); + + const MIN_NBR_PAGE = 1; + const MAX_NBR_PAGE = 6; + + protected $p = null; + + public function collectData() { + $url = sprintf( + '%sv1/group-posts/group/%s/type/%s?', + self::URI, + $this->getGroup(), + $this->getType() + ); + $cursor = 'c=10'; + $posts = array(); + for ($i = 0; $i < $this->getPages(); ++$i) { + $content = getContents($url . $cursor); + $json = json_decode($content, true); + $posts = array_merge($posts, $json['data']['posts']); + $cursor = $json['data']['nextCursor']; + } + + foreach ($posts as $post) { + $item['uri'] = $post['url']; + $item['title'] = $post['title']; + $item['content'] = self::getContent($post); + $item['categories'] = self::getCategories($post); + $item['timestamp'] = self::getTimestamp($post); + + $this->items[] = $item; + } + } + + public function getName() { + if ($this->getInput('d')) { + $name = sprintf('%s - %s', '9GAG', $this->getParameterKey('d')); + } elseif ($this->getInput('g')) { + $name = sprintf('%s - %s', '9GAG', $this->getParameterKey('g')); + if ($this->getInput('t')) { + $name = sprintf('%s [%s]', $name, $this->getParameterKey('t')); + } + } + if (!empty($name)) { + return $name; + } + + return self::NAME; + } + + public function getURI() { + $uri = $this->getInput('g'); + if ($uri === 'default') { + $uri = $this->getInput('t'); + } + + return self::URI . $uri; + } + + protected function getGroup() { + if ($this->getInput('d')) { + return 'default'; + } + + return $this->getInput('g'); + } + + protected function getType() { + if ($this->getInput('d')) { + return $this->getInput('d'); + } + + return $this->getInput('t'); + } + + protected function getPages() { + if ($this->p === null) { + $value = (int) $this->getInput('p'); + $value = ($value < self::MIN_NBR_PAGE) ? self::MIN_NBR_PAGE : $value; + $value = ($value > self::MAX_NBR_PAGE) ? self::MAX_NBR_PAGE : $value; + + $this->p = $value; + } + + return $this->p; + } + + protected function getParameterKey($input = '') { + $params = $this->getParameters(); + $tab = 'Sections'; + if ($input === 'd') { + $tab = 'Popular'; + } + if (!isset($params[$tab][$input])) { + return ''; + } + + return array_search( + $this->getInput($input), + $params[$tab][$input]['values'] + ); + } + + protected static function getContent($post) { + if ($post['type'] === 'Animated') { + $content = self::getAnimated($post); + } elseif ($post['type'] === 'Article') { + $content = self::getArticle($post); + } else { + $content = self::getPhoto($post); + } + + return $content; + } + + protected static function getPhoto($post) { + $image = $post['images']['image460']; + $photo = '<picture>'; + $photo .= sprintf( + '<source srcset="%s" type="image/webp">', + $image['webpUrl'] + ); + $photo .= sprintf( + '<img src="%s" alt="%s" %s>', + $image['url'], + $post['title'], + 'width="500"' + ); + $photo .= '</picture>'; + + return $photo; + } + + protected static function getAnimated($post) { + $poster = $post['images']['image460']['url']; + $sources = $post['images']; + $video = sprintf( + '<video poster="%s" %s>', + $poster, + 'preload="auto" loop controls style="min-height: 300px" width="500"' + ); + $video .= sprintf( + '<source src="%s" type="video/webm">', + $sources['image460sv']['vp9Url'] + ); + $video .= sprintf( + '<source src="%s" type="video/mp4">', + $sources['image460sv']['h265Url'] + ); + $video .= sprintf( + '<source src="%s" type="video/mp4">', + $sources['image460svwm']['url'] + ); + $video .= '</video>'; + + return $video; + } + + protected static function getArticle($post) { + $blocks = $post['article']['blocks']; + $medias = $post['article']['medias']; + $contents = array(); + foreach ($blocks as $block) { + if ('Media' === $block['type']) { + $mediaId = $block['mediaId']; + $contents[] = self::getContent($medias[$mediaId]); + } elseif ('RichText' === $block['type']) { + $contents[] = self::getRichText($block['content']); + } + } + + $content = join('</div><div>', $contents); + $content = sprintf( + '<%1$s>%2$s</%1$s>', + 'div', + $content + ); + + return $content; + } + + protected static function getRichText($text = '') { + $text = trim($text); + + if (preg_match('/^>\s(?<text>.*)/', $text, $matches)) { + $text = sprintf( + '<%1$s>%2$s</%1$s>', + 'blockquote', + $matches['text'] + ); + } else { + $text = sprintf( + '<%1$s>%2$s</%1$s>', + 'p', + $text + ); + } + + return $text; + } + + protected static function getCategories($post) { + $params = self::PARAMETERS; + $sections = $params['Sections']['g']['values']; + + if(isset($post['sections'])) { + $postSections = $post['sections']; + } elseif (isset($post['postSection'])) { + $postSections = array($post['postSection']); + } else { + $postSections = array(); + } + + foreach ($postSections as $key => $section) { + $postSections[$key] = array_search($section, $sections); + } + + return $postSections; + } + + protected static function getTimestamp($post) { + $url = $post['images']['image460']['url']; + $headers = get_headers($url, true); + $date = $headers['Date']; + $time = strtotime($date); + + return $time; + } +} diff --git a/bridges/NotAlwaysBridge.php b/bridges/NotAlwaysBridge.php index f5efff4..b2f4c35 100644 --- a/bridges/NotAlwaysBridge.php +++ b/bridges/NotAlwaysBridge.php @@ -26,6 +26,10 @@ class NotAlwaysBridge extends BridgeAbstract { ) )); + public function getIcon() { + return self::URI . 'favicon_nar.png'; + } + public function collectData(){ $html = getSimpleHTMLDOM($this->getURI()) or returnServerError('Could not request NotAlways.'); diff --git a/bridges/NyaaTorrentsBridge.php b/bridges/NyaaTorrentsBridge.php new file mode 100644 index 0000000..b40b0f9 --- /dev/null +++ b/bridges/NyaaTorrentsBridge.php @@ -0,0 +1,131 @@ +<?php +class NyaaTorrentsBridge extends BridgeAbstract { + + const MAINTAINER = 'ORelio'; + const NAME = 'NyaaTorrents'; + const URI = 'https://nyaa.si/'; + const DESCRIPTION = 'Returns the newest torrents, with optional search criteria.'; + const PARAMETERS = array( + array( + 'f' => array( + 'name' => 'Filter', + 'type' => 'list', + 'values' => array( + 'No filter' => '0', + 'No remakes' => '1', + 'Trusted only' => '2' + ) + ), + 'c' => array( + 'name' => 'Category', + 'type' => 'list', + 'values' => array( + 'All categories' => '0_0', + 'Anime' => '1_0', + 'Anime - AMV' => '1_1', + 'Anime - English' => '1_2', + 'Anime - Non-English' => '1_3', + 'Anime - Raw' => '1_4', + 'Audio' => '2_0', + 'Audio - Lossless' => '2_1', + 'Audio - Lossy' => '2_2', + 'Literature' => '3_0', + 'Literature - English' => '3_1', + 'Literature - Non-English' => '3_2', + 'Literature - Raw' => '3_3', + 'Live Action' => '4_0', + 'Live Action - English' => '4_1', + 'Live Action - Idol/PV' => '4_2', + 'Live Action - Non-English' => '4_3', + 'Live Action - Raw' => '4_4', + 'Pictures' => '5_0', + 'Pictures - Graphics' => '5_1', + 'Pictures - Photos' => '5_2', + 'Software' => '6_0', + 'Software - Apps' => '6_1', + 'Software - Games' => '6_2', + ) + ), + 'q' => array( + 'name' => 'Keyword', + 'description' => 'Keyword(s)', + 'type' => 'text' + ) + ) + ); + + public function getIcon() { + return self::URI . 'static/favicon.png'; + } + + public function collectData() { + + // Build Search URL from user-provided parameters + $search_url = self::URI . '?s=id&o=desc&' + . http_build_query(array( + 'f' => $this->getInput('f'), + 'c' => $this->getInput('c'), + 'q' => $this->getInput('q') + )); + + // Retrieve torrent listing from search results, which does not contain torrent description + $html = getSimpleHTMLDOM($search_url) + or returnServerError('Could not request Nyaa: ' . $search_url); + $links = $html->find('a'); + $results = array(); + foreach ($links as $link) + if (strpos($link->href, '/view/') === 0 && !in_array($link->href, $results)) + $results[] = $link->href; + if (empty($results) && empty($this->getInput('q'))) + returnServerError('No results from Nyaa: ' . $url, 500); + + //Process each item individually + foreach ($results as $element) { + + //Limit total amount of requests + if(count($this->items) >= 20) { + break; + } + + $torrent_id = str_replace('/view/', '', $element); + + //Ignore entries without valid torrent ID + if ($torrent_id != 0 && ctype_digit($torrent_id)) { + + //Retrieve data for this torrent ID + $item_uri = self::URI . 'view/' . $torrent_id; + + //Retrieve full description from torrent page + if ($item_html = getSimpleHTMLDOMCached($item_uri)) { + + //Retrieve data from page contents + $item_title = str_replace(' :: Nyaa', '', $item_html->find('title', 0)->plaintext); + $item_desc = str_get_html(markdownToHtml($item_html->find('#torrent-description', 0)->innertext)); + $item_author = extractFromDelimiters($item_html->outertext, 'href="/user/', '"'); + $item_date = intval(extractFromDelimiters($item_html->outertext, 'data-timestamp="', '"')); + + //Retrieve image for thumbnail or generic logo fallback + $item_image = $this->getURI() . 'static/img/avatar/default.png'; + foreach ($item_desc->find('img') as $img) { + if (strpos($img->src, 'prez') === false) { + $item_image = $img->src; + break; + } + } + + //Build and add final item + $item = array(); + $item['uri'] = $item_uri; + $item['title'] = $item_title; + $item['author'] = $item_author; + $item['timestamp'] = $item_date; + $item['enclosures'] = array($item_image); + $item['content'] = $item_desc; + $this->items[] = $item; + } + } + $element = null; + } + $results = null; + } +} diff --git a/bridges/OnVaSortirBridge.php b/bridges/OnVaSortirBridge.php new file mode 100644 index 0000000..2fcde70 --- /dev/null +++ b/bridges/OnVaSortirBridge.php @@ -0,0 +1,130 @@ +<?php +class OnVaSortirBridge extends FeedExpander { + const MAINTAINER = 'AntoineTurmel'; + const NAME = 'OnVaSortir'; + const URI = 'https://www.onvasortir.com'; + const DESCRIPTION = 'Returns the newest events from OnVaSortir (full text)'; + const PARAMETERS = array( + array( + 'city' => array( + 'name' => 'City', + 'type' => 'list', + 'required' => true, + 'values' => array( + 'Agen' => 'Agen', + 'Ajaccio' => 'Ajaccio', + 'Albi' => 'Albi', + 'Amiens' => 'Amiens', + 'Angers' => 'Angers', + 'Angoulême' => 'Angouleme', + 'Annecy' => 'annecy', + 'Aurillac' => 'aurillac', + 'Auxerre' => 'auxerre', + 'Avignon' => 'avignon', + 'Béziers' => 'Beziers', + 'Bastia' => 'Bastia', + 'Beauvais' => 'Beauvais', + 'Belfort' => 'Belfort', + 'Bergerac' => 'bergerac', + 'Besançon' => 'Besancon', + 'Biarritz' => 'Biarritz', + 'Blois' => 'Blois', + 'Bordeaux' => 'bordeaux', + 'Bourg-en-Bresse' => 'bourg-en-bresse', + 'Bourges' => 'Bourges', + 'Brest' => 'Brest', + 'Brive' => 'brive-la-gaillarde', + 'Bruxelles' => 'bruxelles', + 'Caen' => 'Caen', + 'Calais' => 'Calais', + 'Carcassonne' => 'Carcassonne', + 'Châteauroux' => 'Chateauroux', + 'Chalon-sur-saone' => 'chalon-sur-saone', + 'Chambéry' => 'chambery', + 'Chantilly' => 'chantilly', + 'Charleroi' => 'charleroi', + 'Charleville-Mézières' => 'Charleville-Mezieres', + 'Chartres' => 'Chartres', + 'Cherbourg' => 'Cherbourg', + 'Cholet' => 'cholet', + 'Clermont-Ferrand' => 'Clermont-Ferrand', + 'Compiègne' => 'compiegne', + 'Dieppe' => 'dieppe', + 'Dijon' => 'Dijon', + 'Dunkerque' => 'Dunkerque', + 'Evreux' => 'evreux', + 'Fréjus' => 'frejus', + 'Gap' => 'gap', + 'Genève' => 'geneve', + 'Grenoble' => 'Grenoble', + 'La Roche sur Yon' => 'La-Roche-sur-Yon', + 'La Rochelle' => 'La-Rochelle', + 'Lausanne' => 'lausanne', + 'Laval' => 'Laval', + 'Le Havre' => 'le-havre', + 'Le Mans' => 'le-mans', + 'Liège' => 'liege', + 'Lille' => 'lille', + 'Limoges' => 'Limoges', + 'Lorient' => 'Lorient', + 'Luxembourg' => 'Luxembourg', + 'Lyon' => 'lyon', + 'Marseille' => 'marseille', + 'Metz' => 'Metz', + 'Mons' => 'Mons', + 'Mont de Marsan' => 'mont-de-marsan', + 'Montauban' => 'Montauban', + 'Montluçon' => 'montlucon', + 'Montpellier' => 'montpellier', + 'Mulhouse' => 'Mulhouse', + 'Nîmes' => 'nimes', + 'Namur' => 'Namur', + 'Nancy' => 'Nancy', + 'Nantes' => 'nantes', + 'Nevers' => 'nevers', + 'Nice' => 'nice', + 'Niort' => 'niort', + 'Orléans' => 'orleans', + 'Périgueux' => 'perigueux', + 'Paris' => 'paris', + 'Pau' => 'Pau', + 'Perpignan' => 'Perpignan', + 'Poitiers' => 'Poitiers', + 'Quimper' => 'Quimper', + 'Reims' => 'Reims', + 'Rennes' => 'Rennes', + 'Roanne' => 'roanne', + 'Rodez' => 'rodez', + 'Rouen' => 'Rouen', + 'Saint-Brieuc' => 'Saint-Brieuc', + 'Saint-Etienne' => 'saint-etienne', + 'Saint-Malo' => 'saint-malo', + 'Saint-Nazaire' => 'saint-nazaire', + 'Saint-Quentin' => 'saint-quentin', + 'Saintes' => 'saintes', + 'Strasbourg' => 'Strasbourg', + 'Tarbes' => 'Tarbes', + 'Toulon' => 'Toulon', + 'Toulouse' => 'Toulouse', + 'Tours' => 'Tours', + 'Troyes' => 'troyes', + 'Valence' => 'valence', + 'Vannes' => 'vannes', + 'Zurich' => 'zurich', + ), + 'defaultValue' => '' + ) + ) + ); + protected function parseItem($item){ + $item = parent::parseItem($item); + $html = getSimpleHTMLDOMCached($item['uri']); + $text = $html->find('div.corpsMax', 0)->innertext; + $item['content'] = utf8_encode($text); + return $item; + } + public function collectData(){ + $this->collectExpandableDatas('https://' . + $this->getInput('city') . '.onvasortir.com/rss.php'); + } +} diff --git a/bridges/PikabuBridge.php b/bridges/PikabuBridge.php new file mode 100644 index 0000000..af603ac --- /dev/null +++ b/bridges/PikabuBridge.php @@ -0,0 +1,100 @@ +<?php +class PikabuBridge extends BridgeAbstract { + + const NAME = 'Пикабу'; + const URI = 'https://pikabu.ru'; + const DESCRIPTION = 'Выводит посты по тегу'; + const MAINTAINER = 'em92'; + + const PARAMETERS = array( + 'По тегу' => array( + 'tag' => array( + 'name' => 'Тег', + 'exampleValue' => 'it', + 'required' => true + ), + 'filter' => array( + 'name' => 'Фильтр', + 'type' => 'list', + 'values' => array( + 'Горячее' => 'hot', + 'Свежее' => 'new', + ), + 'defaultValue' => 'hot' + ) + ) + ); + + public function getURI() { + if ($this->getInput('tag')) { + return self::URI . '/tag/' . rawurlencode($this->getInput('tag')) . '/' . rawurlencode($this->getInput('filter')); + } else { + return parent::getURI(); + } + } + + public function getIcon() { + return 'https://cs.pikabu.ru/assets/favicon.ico'; + } + + public function getName() { + if (is_string($this->getInput('tag'))) { + return $this->getInput('tag') . ' - ' . parent::getName(); + } else { + return parent::getName(); + } + } + + public function collectData(){ + $link = $this->getURI(); + + $text_html = getContents($link) or returnServerError('Could not fetch ' . $link); + $text_html = iconv('windows-1251', 'utf-8', $text_html); + $html = str_get_html($text_html); + + foreach($html->find('article.story') as $post) { + $time = $post->find('time.story__datetime', 0); + if (is_null($time)) continue; + + $el_to_remove_selectors = array( + '.story__read-more', + 'svg.story-image__stretch', + ); + + foreach($el_to_remove_selectors as $el_to_remove_selector) { + foreach($post->find($el_to_remove_selector) as $el) { + $el->outertext = ''; + } + } + + foreach($post->find('img') as $img) { + $src = $img->getAttribute('src'); + if (!$src) { + $src = $img->getAttribute('data-src'); + if (!$src) { + continue; + } + } + $img->outertext = '<img src="' . $src . '">'; + } + + $categories = array(); + foreach($post->find('.tags__tag') as $tag) { + if ($tag->getAttribute('data-tag')) { + $categories[] = $tag->innertext; + } + } + + $title = $post->find('.story__title-link', 0); + + $item = array(); + $item['categories'] = $categories; + $item['author'] = $post->find('.user__nick', 0)->innertext; + $item['title'] = $title->plaintext; + $item['content'] = strip_tags(backgroundToImg($post->find('.story__content-inner', 0)->innertext), '<br><p><img>'); + $item['uri'] = $title->href; + $item['timestamp'] = strtotime($time->getAttribute('datetime')); + $this->items[] = $item; + } + } +} diff --git a/bridges/PinterestBridge.php b/bridges/PinterestBridge.php index d2dd890..2917b26 100644 --- a/bridges/PinterestBridge.php +++ b/bridges/PinterestBridge.php @@ -25,6 +25,10 @@ class PinterestBridge extends FeedExpander { ) ); + public function getIcon() { + return 'https://s.pinimg.com/webapp/style/images/favicon-9f8f9adf.png'; + } + public function collectData(){ switch($this->queriedContext) { case 'By username and board': diff --git a/bridges/PixivBridge.php b/bridges/PixivBridge.php index 3a4cc93..4e4cf65 100644 --- a/bridges/PixivBridge.php +++ b/bridges/PixivBridge.php @@ -18,7 +18,7 @@ class PixivBridge extends BridgeAbstract { public function collectData(){ - $html = getContents(static::URI.'search.php?word=' . urlencode($this->getInput('tag'))) + $html = getContents(static::URI . 'search.php?word=' . urlencode($this->getInput('tag'))) or returnClientError('Unable to query pixiv.net'); $regex = '/<input type="hidden"id="js-mount-point-search-result-list"data-items="([^"]*)/'; $timeRegex = '/img\/([0-9]{4})\/([0-9]{2})\/([0-9]{2})\/([0-9]{2})\/([0-9]{2})\/([0-9]{2})\//'; @@ -40,7 +40,8 @@ class PixivBridge extends BridgeAbstract { preg_match_all($timeRegex, $result['url'], $dt, PREG_SET_ORDER, 0); $elementDate = DateTime::createFromFormat('YmdHis', - $dt[0][1] . $dt[0][2] . $dt[0][3] . $dt[0][4] . $dt[0][5] . $dt[0][6]); + $dt[0][1] . $dt[0][2] . $dt[0][3] . $dt[0][4] . $dt[0][5] . $dt[0][6], + new DateTimeZone('Asia/Tokyo')); $item['timestamp'] = $elementDate->getTimestamp(); $item['content'] = "<img src='" . $this->cacheImage($result['url'], $item['id']) . "' />"; @@ -48,11 +49,11 @@ class PixivBridge extends BridgeAbstract { } } - public function cacheImage($url, $illustId) { + private function cacheImage($url, $illustId) { $url = str_replace('_master1200', '', $url); $url = str_replace('c/240x240/img-master/', 'img-original/', $url); - $path = CACHE_DIR . '/pixiv_img'; + $path = PATH_CACHE . '/pixiv_img'; if(!is_dir($path)) mkdir($path, 0755, true); diff --git a/bridges/RTBFBridge.php b/bridges/RTBFBridge.php index 22cdaf4..0f0acdc 100644 --- a/bridges/RTBFBridge.php +++ b/bridges/RTBFBridge.php @@ -58,7 +58,7 @@ class RTBFBridge extends BridgeAbstract { public function getName(){ if(!is_null($this->getInput('c'))) { - return $this->getInput('c') .' - RTBF Bridge'; + return $this->getInput('c') . ' - RTBF Bridge'; } return parent::getName(); diff --git a/bridges/RadioMelodieBridge.php b/bridges/RadioMelodieBridge.php index 9b3772b..ca033fd 100644 --- a/bridges/RadioMelodieBridge.php +++ b/bridges/RadioMelodieBridge.php @@ -5,6 +5,10 @@ class RadioMelodieBridge extends BridgeAbstract { const DESCRIPTION = 'Retourne les actualités publiées par Radio Melodie'; const MAINTAINER = 'sysadminstory'; + public function getIcon() { + return self::URI . 'img/favicon.png'; + } + public function collectData(){ $html = getSimpleHTMLDOM(self::URI . 'actu') or returnServerError('Could not request Radio Melodie.'); @@ -23,7 +27,7 @@ class RadioMelodieBridge extends BridgeAbstract { $item['enclosures'] = array($pictureURL); $item['uri'] = self::URI . $element->parent()->href; $item['title'] = $element->find('h3', 0)->plaintext; - $item['content'] = $element->find('p', 0)->plaintext . '<br/><img src="'.$pictureURL.'"/>'; + $item['content'] = $element->find('p', 0)->plaintext . '<br/><img src="' . $pictureURL . '"/>'; $this->items[] = $item; } } diff --git a/bridges/RainbowSixSiegeBridge.php b/bridges/RainbowSixSiegeBridge.php index d362bbd..724edc8 100644 --- a/bridges/RainbowSixSiegeBridge.php +++ b/bridges/RainbowSixSiegeBridge.php @@ -7,10 +7,14 @@ class RainbowSixSiegeBridge extends BridgeAbstract { const CACHE_TIMEOUT = 7200; // 2h const DESCRIPTION = 'Latest articles from the Rainbow Six Siege blog'; + public function getIcon() { + return 'https://ubistatic19-a.akamaihd.net/resource/en-us/game/rainbow6/siege-v3/r6s-favicon_316592.ico'; + } + public function collectData(){ $dlUrl = 'https://prod-tridionservice.ubisoft.com/live/v1/News/Latest?templateId=tcm%3A152-7677'; - $dlUrl .= '8-32&pageIndex=0&pageSize=10&language=en-US&detailPageId=tcm%3A152-194572-64'; - $dlUrl .= '&keywordList=175426&siteId=undefined&useSeoFriendlyUrl=true'; + $dlUrl .= '8-32&pageIndex=0&pageSize=10&language=en-US&detailPageId=tcm%3A150-194572-64'; + $dlUrl .= '&keywordList=233416%2C316144%2C233418%2C233417&siteId=undefined&useSeoFriendlyUrl=true'; $jsonString = getContents($dlUrl) or returnServerError('Error while downloading the website content'); $json = json_decode($jsonString, true); diff --git a/bridges/Releases3DSBridge.php b/bridges/Releases3DSBridge.php index a7e1778..6c159d1 100644 --- a/bridges/Releases3DSBridge.php +++ b/bridges/Releases3DSBridge.php @@ -9,16 +9,6 @@ class Releases3DSBridge extends BridgeAbstract { public function collectData(){ - function extractFromDelimiters($string, $start, $end){ - if(strpos($string, $start) !== false) { - $section_retrieved = substr($string, strpos($string, $start) + strlen($start)); - $section_retrieved = substr($section_retrieved, 0, strpos($section_retrieved, $end)); - return $section_retrieved; - } - - return false; - } - function typeToString($type){ switch($type) { case 1: return '3DS Game'; @@ -76,8 +66,8 @@ class Releases3DSBridge extends BridgeAbstract { $ignDate = time(); $ignCoverArt = ''; - $ignSearchUrl = 'http://www.ign.com/search?q=' . urlencode($name); - if($ignResult = getSimpleHTMLDOM($ignSearchUrl)) { + $ignSearchUrl = 'https://www.ign.com/search?q=' . urlencode($name); + if($ignResult = getSimpleHTMLDOMCached($ignSearchUrl)) { $ignCoverArt = $ignResult->find('div.search-item-media', 0)->find('img', 0)->src; $ignDesc = $ignResult->find('div.search-item-description', 0)->plaintext; $ignLink = $ignResult->find('div.search-item-sub-title', 0)->find('a', 1)->href; @@ -127,6 +117,7 @@ class Releases3DSBridge extends BridgeAbstract { $item['title'] = $name; $item['author'] = $publisher; $item['timestamp'] = $ignDate; + $item['enclosures'] = array($ignCoverArt); $item['uri'] = empty($ignLink) ? $searchLinkDuckDuckGo : $ignLink; $item['content'] = $ignDescription . $releaseDescription . $releaseSearchLinks; $this->items[] = $item; diff --git a/bridges/Rue89Bridge.php b/bridges/Rue89Bridge.php index 72f01eb..ca0e9b6 100644 --- a/bridges/Rue89Bridge.php +++ b/bridges/Rue89Bridge.php @@ -1,25 +1,50 @@ <?php -class Rue89Bridge extends FeedExpander { +class Rue89Bridge extends BridgeAbstract { - const MAINTAINER = 'pit-fgfjiudghdf'; + const MAINTAINER = 'teromene'; const NAME = 'Rue89'; - const URI = 'http://rue89.nouvelobs.com/'; - const DESCRIPTION = 'Returns the 5 newest posts from Rue89 (full text)'; + const URI = 'https://www.nouvelobs.com/rue89/'; + const DESCRIPTION = 'Returns the newest posts from Rue89'; - protected function parseItem($item){ - $item = parent::parseItem($item); - $url = 'http://api.rue89.nouvelobs.com/export/mobile2/node/' - . str_replace(' ', '', substr($item['uri'], -8)) - . '/full'; + public function collectData() { - $datas = json_decode(getContents($url), true); - $item['content'] = $datas['node']['body']; + $jsonArticles = getContents('https://appdata.nouvelobs.com/rue89/feed.json') + or die('Unable to query Rue89 !'); + $articles = json_decode($jsonArticles)->items; + foreach($articles as $article) { + $this->items[] = $this->getArticle($article); + } - return $item; } - public function collectData(){ - $this->collectExpandableDatas('http://api.rue89.nouvelobs.com/feed'); + private function getArticle($articleInfo) { + + $articleJson = getContents($articleInfo->json_url) or die('Unable to get article !'); + $article = json_decode($articleJson); + $item = array(); + $item['title'] = $article->title; + $item['uri'] = $article->url; + if($article->content_premium !== null) { + $item['content'] = $article->content_premium; + } else { + $item['content'] = $article->content; + } + $item['timestamp'] = $article->date_publi; + $item['author'] = $article->author->show_name; + + $item['enclosures'] = array(); + foreach($article->images as $image) { + $item['enclosures'][] = $image->url; + } + + $item['categories'] = array(); + foreach($article->categories as $category) { + $item['categories'][] = $category->title; + } + + return $item; + } + } diff --git a/bridges/SexactuBridge.php b/bridges/SexactuBridge.php deleted file mode 100644 index b0a7174..0000000 --- a/bridges/SexactuBridge.php +++ /dev/null @@ -1,88 +0,0 @@ -<?php -class SexactuBridge extends BridgeAbstract { - - const MAINTAINER = 'Riduidel'; - const NAME = 'Sexactu'; - const AUTHOR = 'Maïa Mazaurette'; - const URI = 'http://www.gqmagazine.fr'; - const CACHE_TIMEOUT = 7200; // 2h - const DESCRIPTION = 'Sexactu via rss-bridge'; - - const REPLACED_ATTRIBUTES = array( - 'href' => 'href', - 'src' => 'src', - 'data-original' => 'src' - ); - - public function getURI(){ - return self::URI . '/sexactu'; - } - - public function collectData(){ - $html = getSimpleHTMLDOM($this->getURI()) - or returnServerError('Could not request ' . $this->getURI()); - - $sexactu = $html->find('.container_sexactu', 0); - $rowList = $sexactu->find('.row'); - foreach($rowList as $row) { - // only use first list as second one only contains pages numbers - - $title = $row->find('.title', 0); - if($title) { - $item = array(); - $item['author'] = self::AUTHOR; - $item['title'] = $title->plaintext; - $urlAttribute = 'data-href'; - $uri = $title->$urlAttribute; - if($uri === false) - continue; - if(substr($uri, 0, 1) === 'h') { // absolute uri - $item['uri'] = $uri; - } else if(substr($uri, 0, 1) === '/') { // domain relative url - $item['uri'] = self::URI . $uri; - } else { - $item['uri'] = $this->getURI() . $uri; - } - $article = $this->loadFullArticle($item['uri']); - $item['content'] = $this->replaceUriInHtmlElement($article->find('.article_content', 0)); - - $publicationDate = $article->find('time[itemprop=datePublished]', 0); - $short_date = $publicationDate->datetime; - $item['timestamp'] = strtotime($short_date); - } else { - // Sometimes we get rubbish, ignore. - continue; - } - $this->items[] = $item; - } - } - - /** - * Loads the full article and returns the contents - * @param $uri The article URI - * @return The article content - */ - private function loadFullArticle($uri){ - $html = getSimpleHTMLDOMCached($uri); - - $content = $html->find('#article', 0); - if($content) { - return $content; - } - - return null; - } - - /** - * Replaces all relative URIs with absolute ones - * @param $element A simplehtmldom element - * @return The $element->innertext with all URIs replaced - */ - private function replaceUriInHtmlElement($element){ - $returned = $element->innertext; - foreach (self::REPLACED_ATTRIBUTES as $initial => $final) { - $returned = str_replace($initial . '="/', $final . '="' . self::URI . '/', $returned); - } - return $returned; - } -} diff --git a/bridges/SkimfeedBridge.php b/bridges/SkimfeedBridge.php new file mode 100644 index 0000000..0b22b11 --- /dev/null +++ b/bridges/SkimfeedBridge.php @@ -0,0 +1,825 @@ +<?php + +class SkimfeedBridge extends BridgeAbstract { + + const CONTEXT_NEWS_BOX = 'News box'; + const CONTEXT_HOT_TOPICS = 'Hot topics'; + const CONTEXT_TECH_NEWS = 'Tech news'; + const CONTEXT_CUSTOM = 'Custom feed'; + + const NAME = 'Skimfeed Bridge'; + const URI = 'https://skimfeed.com'; + const DESCRIPTION = 'Returns feeds from Skimfeed, also supports custom feeds!'; + const MAINTAINER = 'logmanoriginal'; + const CACHE_TIMEOUT = 3600; + + const PARAMETERS = array( + self::CONTEXT_NEWS_BOX => array( // auto-generated (see below) + 'box_channel' => array( + 'name' => 'Channel', + 'type' => 'list', + 'required' => true, + 'title' => 'Select your channel', + 'values' => array( + 'Hacker News' => '/news/hacker-news.html', + 'QZ' => '/news/qz.html', + 'The Verge' => '/news/the-verge.html', + 'Slashdot' => '/news/slashdot.html', + 'Lifehacker' => '/news/lifehacker.html', + 'Gizmag' => '/news/gizmag.html', + 'Fast Company' => '/news/fast-company.html', + 'Engadget' => '/news/engadget.html', + 'Wired' => '/news/wired.html', + 'MakeUseOf' => '/news/makeuseof.html', + 'Techcrunch' => '/news/techcrunch.html', + 'Apple Insider' => '/news/apple-insider.html', + 'ArsTechnica' => '/news/arstechnica.html', + 'Tech in Asia' => '/news/tech-in-asia.html', + 'FastCoExist' => '/news/fastcoexist.html', + 'Digital Trends' => '/news/digital-trends.html', + 'AnandTech' => '/news/anandtech.html', + 'How to Geek' => '/news/how-to-geek.html', + 'Geek' => '/news/geek.html', + 'BBC Technology' => '/news/bbc-technology.html', + 'Extreme Tech' => '/news/extreme-tech.html', + 'Packet Storm Sec' => '/news/packet-storm-sec.html', + 'MedGadget' => '/news/medgadget.html', + 'Design' => '/news/design.html', + 'The Next Web' => '/news/the-next-web.html', + 'Bit-Tech' => '/news/bit-tech.html', + 'Next Big Future' => '/news/next-big-future.html', + 'A VC' => '/news/a-vc.html', + 'Copyblogger' => '/news/copyblogger.html', + 'Smashing Mag' => '/news/smashing-mag.html', + 'Continuations' => '/news/continuations.html', + 'Cult of Mac' => '/news/cult-of-mac.html', + 'SecuriTeam' => '/news/securiteam.html', + 'The Tech Block' => '/news/the-tech-block.html', + 'BetaBeat' => '/news/betabeat.html', + 'PC Mag' => '/news/pc-mag.html', + 'Venture Beat' => '/news/venture-beat.html', + 'ReadWriteWeb' => '/news/readwriteweb.html', + 'High Scalability' => '/news/high-scalability.html', + ) + ) + ), + self::CONTEXT_HOT_TOPICS => array(), + self::CONTEXT_TECH_NEWS => array( // auto-generated (see below) + 'tech_channel' => array( + 'name' => 'Tech channel', + 'type' => 'list', + 'required' => true, + 'title' => 'Select your tech channel', + 'values' => array( + 'Agg' => array( + 'Reddit' => '/news/reddit.html', + 'Tech Insider' => '/news/tech-insider.html', + 'Digg' => '/news/digg.html', + 'Meta Filter' => '/news/meta-filter.html', + 'Fark' => '/news/fark.html', + 'Mashable' => '/news/mashable.html', + 'Ad Week' => '/news/ad-week.html', + 'The Chive' => '/news/the-chive.html', + 'BoingBoing' => '/news/boingboing.html', + 'Vice' => '/news/vice.html', + 'ClientsFromHell' => '/news/clientsfromhell.html', + 'How Stuff Works' => '/news/how-stuff-works.html', + 'Buzzfeed' => '/news/buzzfeed.html', + 'BoingBoing' => '/news/boingboing.html', + 'Cracked' => '/news/cracked.html', + 'Weird News' => '/news/weird-news.html', + 'ITOTD' => '/news/itotd.html', + 'Metafilter' => '/news/metafilter.html', + 'TheOnion' => '/news/theonion.html', + ), + 'Cars' => array( + 'Reddit Cars' => '/news/reddit-cars.html', + 'NYT Auto' => '/news/nyt-auto.html', + 'Truth About Cars' => '/news/truth-about-cars.html', + 'AutoBlog' => '/news/autoblog.html', + 'AutoSpies' => '/news/autospies.html', + 'Autoweek' => '/news/autoweek.html', + 'The Garage' => '/news/the-garage.html', + 'Car and Driver' => '/news/car-and-driver.html', + 'EGM Car Tech' => '/news/egm-car-tech.html', + 'Top Gear' => '/news/top-gear.html', + 'eGarage' => '/news/egarage.html', + ), + 'Comics' => array( + 'Penny Arcade' => '/news/penny-arcade.html', + 'XKCD' => '/news/xkcd.html', + 'Channelate' => '/news/channelate.html', + 'Savage Chicken' => '/news/savage-chicken.html', + 'Dinosaur Comics' => '/news/dinosaur-comics.html', + 'Explosm' => '/news/explosm.html', + 'PoorlyDLines' => '/news/poorlydlines.html', + 'Moonbeard' => '/news/moonbeard.html', + 'Nedroid' => '/news/nedroid.html', + ), + 'Design' => array( + 'FastCoCreate' => '/news/fastcocreate.html', + 'Dezeen' => '/news/dezeen.html', + 'Design Boom' => '/news/design-boom.html', + 'Mmminimal' => '/news/mmminimal.html', + 'We Heart' => '/news/we-heart.html', + 'CreativeBloq' => '/news/creativebloq.html', + 'TheDSGNblog' => '/news/thedsgnblog.html', + 'Grainedit' => '/news/grainedit.html', + ), + 'Football' => array( + 'Mail Football' => '/news/mail-football.html', + 'Yahoo Football' => '/news/yahoo-football.html', + 'FourFourTwo' => '/news/fourfourtwo.html', + 'Goal' => '/news/goal.html', + 'BBC Football' => '/news/bbc-football.html', + 'TalkSport' => '/news/talksport.html', + '101 Great Goals' => '/news/101-great-goals.html', + 'Who Scored' => '/news/who-scored.html', + 'Football365 Champ' => '/news/football365-champ.html', + 'Football365 Premier' => '/news/football365-premier.html', + 'BleacherReport' => '/news/bleacherreport.html', + ), + 'Gaming' => array( + 'Polygon' => '/news/polygon.html', + 'Gamespot' => '/news/gamespot.html', + 'RockPaperShotgun' => '/news/rockpapershotgun.html', + 'VG247' => '/news/vg247.html', + 'IGN' => '/news/ign.html', + 'Reddit Games' => '/news/reddit-games.html', + 'TouchArcade' => '/news/toucharcade.html', + 'GamesRadar' => '/news/gamesradar.html', + 'Siliconera' => '/news/siliconera.html', + 'Reddit GameDeals' => '/news/reddit-gamedeals.html', + 'Joystiq' => '/news/joystiq.html', + 'GameInformer' => '/news/gameinformer.html', + 'PSN Blog' => '/news/psn-blog.html', + 'Reddit GamerNews' => '/news/reddit-gamernews.html', + 'Steam' => '/news/steam.html', + 'DualShockers' => '/news/dualshockers.html', + 'ShackNews' => '/news/shacknews.html', + 'CheapAssGamer' => '/news/cheapassgamer.html', + 'Eurogamer' => '/news/eurogamer.html', + 'Major Nelson' => '/news/major-nelson.html', + 'Reddit Truegaming' => '/news/reddit-truegaming.html', + 'GameTrailers' => '/news/gametrailers.html', + 'GamaSutra' => '/news/gamasutra.html', + 'USGamer' => '/news/usgamer.html', + 'Shoryuken' => '/news/shoryuken.html', + 'Destructoid' => '/news/destructoid.html', + 'ArsGaming' => '/news/arsgaming.html', + 'XBOX Blog' => '/news/xbox-blog.html', + 'GiantBomb' => '/news/giantbomb.html', + 'VideoGamer' => '/news/videogamer.html', + 'Pocket Tactics' => '/news/pocket-tactics.html', + 'WiredGaming' => '/news/wiredgaming.html', + 'AllGamesBeta' => '/news/allgamesbeta.html', + 'OnGamers' => '/news/ongamers.html', + 'Reddit GameBundles' => '/news/reddit-gamebundles.html', + 'Kotaku' => '/news/kotaku.html', + 'PCGamer' => '/news/pcgamer.html', + ), + 'Investing' => array( + 'Seeking Alpha' => '/news/seeking-alpha.html', + 'BBC Business' => '/news/bbc-business.html', + 'Harvard Biz' => '/news/harvard-biz.html', + 'Market Watch' => '/news/market-watch.html', + 'Investor Place' => '/news/investor-place.html', + 'Money Week' => '/news/money-week.html', + 'Moneybeat' => '/news/moneybeat.html', + 'Dealbook' => '/news/dealbook.html', + 'Economist Business' => '/news/economist-business.html', + 'Economist' => '/news/economist.html', + 'Economist CN' => '/news/economist-cn.html', + ), + 'Long' => array( + 'The Atlantic' => '/news/the-atlantic.html', + 'Reddit Long' => '/news/reddit-long.html', + 'Paris Review' => '/news/paris-review.html', + 'New Yorker' => '/news/new-yorker.html', + 'LongForm' => '/news/longform.html', + 'LongReads' => '/news/longreads.html', + 'The Browser' => '/news/the-browser.html', + 'The Feature' => '/news/the-feature.html', + ), + 'MMA' => array( + 'MMA Weekly' => '/news/mma-weekly.html', + 'MMAFighting' => '/news/mmafighting.html', + 'Reddit MMA' => '/news/reddit-mma.html', + 'Sherdog Articles' => '/news/sherdog-articles.html', + 'FightLand Vice' => '/news/fightland-vice.html', + 'Sherdog Forum' => '/news/sherdog-forum.html', + 'MMA Junkie' => '/news/mma-junkie.html', + 'Sherdog MMA Video' => '/news/sherdog-mma-video.html', + 'BloodyElbow' => '/news/bloodyelbow.html', + 'CageWriter' => '/news/cagewriter.html', + 'Sherdog News' => '/news/sherdog-news.html', + 'MMAForum' => '/news/mmaforum.html', + 'MMA Junkie Radio' => '/news/mma-junkie-radio.html', + 'UFC News' => '/news/ufc-news.html', + 'FightLinker' => '/news/fightlinker.html', + 'Bodybuilding MMA' => '/news/bodybuilding-mma.html', + 'BleacherReport MMA' => '/news/bleacherreport-mma.html', + 'FiveOuncesofPain' => '/news/fiveouncesofpain.html', + 'Sherdog Pictures' => '/news/sherdog-pictures.html', + 'CagePotato' => '/news/cagepotato.html', + 'Sherdog Radio' => '/news/sherdog-radio.html', + 'ProMMARadio' => '/news/prommaradio.html', + ), + 'Mobile' => array( + 'Macrumors' => '/news/macrumors.html', + 'Android Police' => '/news/android-police.html', + 'GSM Arena' => '/news/gsm-arena.html', + 'DigiTrend Mobile' => '/news/digitrend-mobile.html', + 'Mobile Nation' => '/news/mobile-nation.html', + 'TechRadar' => '/news/techradar.html', + 'ZDNET Mobile' => '/news/zdnet-mobile.html', + 'MacWorld' => '/news/macworld.html', + 'Android Dev Blog' => '/news/android-dev-blog.html', + ), + 'News' => array( + 'Daily Mail' => '/news/daily-mail.html', + 'Business Insider' => '/news/business-insider.html', + 'The Guardian' => '/news/the-guardian.html', + 'Fox' => '/news/fox.html', + 'BBC World' => '/news/bbc-world.html', + 'MSNBC' => '/news/msnbc.html', + 'ABC News' => '/news/abc-news.html', + 'Al Jazeera' => '/news/al-jazeera.html', + 'Business Insider India' => '/news/business-insider-india.html', + 'Observer' => '/news/observer.html', + 'NYT Tech' => '/news/nyt-tech.html', + 'NYT World' => '/news/nyt-world.html', + 'CNN' => '/news/cnn.html', + 'Japan Times' => '/news/japan-times.html', + 'WorldCrunch' => '/news/worldcrunch.html', + 'Pro publica' => '/news/pro-publica.html', + 'OZY' => '/news/ozy.html', + 'Times of India' => '/news/times-of-india.html', + 'The Australian' => '/news/the-australian.html', + 'Harpers' => '/news/harpers.html', + 'Moscow Times' => '/news/moscow-times.html', + 'The Times' => '/news/the-times.html', + 'Reuters Tech' => '/news/reuters-tech.html', + ), + 'Politics' => array( + 'FreeRepublic' => '/news/freerepublic.html', + 'Salon' => '/news/salon.html', + 'DrudgeReport' => '/news/drudgereport.html', + 'TheHill' => '/news/thehill.html', + 'TheBlaze' => '/news/theblaze.html', + 'InfoWars' => '/news/infowars.html', + 'New Republic' => '/news/new-republic.html', + 'WashTimes' => '/news/washtimes.html', + 'RealCleanPol' => '/news/realcleanpol.html', + 'Fact Check' => '/news/fact-check.html', + 'DailyKos' => '/news/dailykos.html', + 'NewsMax' => '/news/newsmax.html', + 'Politico' => '/news/politico.html', + 'Michelle Malkin' => '/news/michelle-malkin.html', + ), + 'Reddit' => array( + 'R Movies' => '/news/r-movies.html', + 'R News' => '/news/r-news.html', + 'Futurology' => '/news/futurology.html', + 'R All' => '/news/r-all.html', + 'R Music' => '/news/r-music.html', + 'R Askscience' => '/news/r-askscience.html', + 'R Technology' => '/news/r-technology.html', + 'R Bestof' => '/news/r-bestof.html', + 'R Askreddit' => '/news/r-askreddit.html', + 'R Worldnews' => '/news/r-worldnews.html', + 'R Explainlikeimfive' => '/news/r-explainlikeimfive.html', + 'R Iama' => '/news/r-iama.html', + ), + 'Science' => array( + 'PhysOrg' => '/news/physorg.html', + 'Hack-a-day' => '/news/hack-a-day.html', + 'Reddit Science' => '/news/reddit-science.html', + 'Stats Blog' => '/news/stats-blog.html', + 'Flowing Data' => '/news/flowing-data.html', + 'Eureka Alert' => '/news/eureka-alert.html', + 'Robotics BizRev' => '/news/robotics-bizrev.html', + 'Planet big Data' => '/news/planet-big-data.html', + 'Makezine' => '/news/makezine.html', + 'MIT Tech' => '/news/mit-tech.html', + 'R Bloggers' => '/news/r-bloggers.html', + 'DataIsBeautiful' => '/news/dataisbeautiful.html', + 'Ted Videos' => '/news/ted-videos.html', + 'Advanced Science' => '/news/advanced-science.html', + 'Robotiq' => '/news/robotiq.html', + 'Science Daily' => '/news/science-daily.html', + 'IEEE Robotics' => '/news/ieee-robotics.html', + 'PSFK' => '/news/psfk.html', + 'Discover Magazine' => '/news/discover-magazine.html', + 'DataTau' => '/news/datatau.html', + 'RoboHub' => '/news/robohub.html', + 'Discovery' => '/news/discovery.html', + 'Smart Data' => '/news/smart-data.html', + 'Whats Big Data' => '/news/whats-big-data.html', + ), + 'Tech' => array( + 'Hacker News' => '/news/hacker-news.html', + 'The Verge' => '/news/the-verge.html', + 'Lifehacker' => '/news/lifehacker.html', + 'Fast Company' => '/news/fast-company.html', + 'ArsTechnica' => '/news/arstechnica.html', + 'MakeUseOf' => '/news/makeuseof.html', + 'FastCoExist' => '/news/fastcoexist.html', + 'How to Geek' => '/news/how-to-geek.html', + 'The Next Web' => '/news/the-next-web.html', + 'Engadget' => '/news/engadget.html', + 'Gizmag' => '/news/gizmag.html', + 'QZ' => '/news/qz.html', + 'Wired' => '/news/wired.html', + 'Techcrunch' => '/news/techcrunch.html', + 'Slashdot' => '/news/slashdot.html', + 'Extreme Tech' => '/news/extreme-tech.html', + 'AnandTech' => '/news/anandtech.html', + 'Digital Trends' => '/news/digital-trends.html', + 'Next Big Future' => '/news/next-big-future.html', + 'Apple Insider' => '/news/apple-insider.html', + 'Geek' => '/news/geek.html', + 'BBC Technology' => '/news/bbc-technology.html', + 'Bit-Tech' => '/news/bit-tech.html', + 'Packet Storm Sec' => '/news/packet-storm-sec.html', + 'Design' => '/news/design.html', + 'High Scalability' => '/news/high-scalability.html', + 'Smashing Mag' => '/news/smashing-mag.html', + 'The Tech Block' => '/news/the-tech-block.html', + 'A VC' => '/news/a-vc.html', + 'Tech in Asia' => '/news/tech-in-asia.html', + 'ReadWriteWeb' => '/news/readwriteweb.html', + 'PC Mag' => '/news/pc-mag.html', + 'Continuations' => '/news/continuations.html', + 'Copyblogger' => '/news/copyblogger.html', + 'Cult of Mac' => '/news/cult-of-mac.html', + 'BetaBeat' => '/news/betabeat.html', + 'MedGadget' => '/news/medgadget.html', + 'SecuriTeam' => '/news/securiteam.html', + 'Venture Beat' => '/news/venture-beat.html', + ), + 'Trend' => array( + 'Trend Hunter' => '/news/trend-hunter.html', + 'ApartmentT' => '/news/apartmentt.html', + 'GQ' => '/news/gq.html', + 'Digital Trends' => '/news/digital-trends.html', + 'Cool Hunting' => '/news/cool-hunting.html', + 'FastCoDesign' => '/news/fastcodesign.html', + 'TC Startups' => '/news/tc-startups.html', + 'Killer Startups' => '/news/killer-startups.html', + 'DigiInfo' => '/news/digiinfo.html', + 'New Startups' => '/news/new-startups.html', + 'DigiTrends' => '/news/digitrends.html', + ), + 'Watches' => array( + 'Hodinkee' => '/news/hodinkee.html', + 'Quill and Pad' => '/news/quill-and-pad.html', + 'Monochrome' => '/news/monochrome.html', + 'Deployant' => '/news/deployant.html', + 'Watches by SJX' => '/news/watches-by-sjx.html', + 'Fratello Watches' => '/news/fratello-watches.html', + 'A Blog to Watch' => '/news/a-blog-to-watch.html', + 'Wound for Life' => '/news/wound-for-life.html', + 'Watch Paper' => '/news/watch-paper.html', + 'Watch Report' => '/news/watch-report.html', + 'Perpetuelle' => '/news/perpetuelle.html', + ), + 'Youtube' => array( + 'LinusTechTips' => '/news/linustechtips.html', + 'MetalJesusRocks' => '/news/metaljesusrocks.html', + 'TotalBiscuit' => '/news/totalbiscuit.html', + 'DexBonus' => '/news/dexbonus.html', + 'Lon Siedman' => '/news/lon-siedman.html', + 'MKBHD' => '/news/mkbhd.html', + 'Terry A Davis' => '/news/terry-a-davis.html', + 'HappyConsole' => '/news/happyconsole.html', + 'Austin Evans' => '/news/austin-evans.html', + 'NCIX' => '/news/ncix.html', + ), + ) + ), + ), + self::CONTEXT_CUSTOM => array( + 'config' => array( + 'name' => 'Configuration', + 'type' => 'text', + 'required' => true, + 'title' => 'Enter feed numbers from Skimfeed!', + 'exampleValue' => '5,8,2,l,p,9,23' + ) + ), + 'global' => array( + 'limit' => array( + 'name' => 'Limit', + 'type' => 'number', + 'title' => 'Limits the number of returned items in the feed', + 'exampleValue' => 10 + ) + ) + ); + + public function getURI() { + + switch($this->queriedContext) { + + case self::CONTEXT_NEWS_BOX: + + $channel = $this->getInput('box_channel'); + + if($channel) { + return static::URI . $channel; + } + + break; + + case self::CONTEXT_HOT_TOPICS: + return static::URI; + + case self::CONTEXT_TECH_NEWS: + + $channel = $this->getInput('tech_channel'); + + if($channel) { + return static::URI . $channel; + } + + break; + + case self::CONTEXT_CUSTOM: + + $config = $this->getInput('config'); + + return static::URI . '/custom.php?f=' . urlencode($config); + + } + + return parent::getURI(); + + } + + public function getName() { + + switch($this->queriedContext) { + + case self::CONTEXT_NEWS_BOX: + + $channel = $this->getInput('box_channel'); + + $title = array_search( + $channel, + static::PARAMETERS[self::CONTEXT_NEWS_BOX]['box_channel']['values'] + ); + + return $title . ' - ' . static::NAME; + + case self::CONTEXT_HOT_TOPICS: + return 'Hot topics - ' . static::NAME; + + case self::CONTEXT_TECH_NEWS: + + $channel = $this->getInput('tech_channel'); + + $titles = array(); + + foreach(static::PARAMETERS[self::CONTEXT_TECH_NEWS]['tech_channel']['values'] as $ch) { + $titles = array_merge($titles, $ch); + } + + $title = array_search($channel, $titles); + + return $title . ' - ' . static::NAME; + + case self::CONTEXT_CUSTOM: + return 'Custom - ' . static::NAME; + + } + + return parent::getName(); + + } + + public function collectData() { + + // enable to export parameter lists + // $this->exportBoxChannels(); die; + // $this->exportTechChannels(); die; + + $html = getSimpleHTMLDOM($this->getURI()) + or returnServerError('Request to ' . $this->getURI() . ' failed!'); + + defaultLinkTo($html, static::URI); + + switch($this->queriedContext) { + + case self::CONTEXT_NEWS_BOX: + + $author = array_search( + $this->getInput('box_channel'), + static::PARAMETERS[self::CONTEXT_NEWS_BOX]['box_channel']['values'] + ); + + $author = '<a href="' + . $this->getURI() + . '">' + . $author + . '</a>'; + + $this->extractFeed($html, $author); + break; + + case self::CONTEXT_HOT_TOPICS: + $this->extractHotTopics($html); + break; + + case self::CONTEXT_TECH_NEWS: + $authors = array(); + + foreach(static::PARAMETERS[self::CONTEXT_TECH_NEWS]['tech_channel']['values'] as $ch) { + $authors = array_merge($authors, $ch); + } + + $author = '<a href="' + . $this->getURI() + . '">' + . array_search($this->getInput('tech_channel'), $authors) + . '</a>'; + + $this->extractFeed($html, $author); + break; + + case self::CONTEXT_CUSTOM: + $this->extractCustomFeed($html); + break; + + } + + } + + private function extractFeed($html, $author) { + + $articles = $html->find('li') + or returnServerError('Could not find articles!'); + + if(count($articles) === 1 + && stristr($articles[0]->plaintext, 'Nothing new in the last 48 hours')) { + return; // Nothing to show + } + + $limit = $this->getInput('limit') ?: -1; + + foreach($articles as $article) { + + $anchor = $article->find('a', 0) + or returnServerError('Could not find anchor!'); + + $item = array(); + + $item['uri'] = $this->getTarget($anchor); + $item['title'] = trim($anchor->plaintext); + + // The timestamp is encoded as relative time (max. the last 48 hours) + // like this: "- 7 hours". It should always be at the end of the article: + $age = substr($article->plaintext, strrpos($article->plaintext, '-')); + + $item['timestamp'] = strtotime($age); + $item['author'] = $author; + + $this->items[] = $item; + + if($limit > 0 && count($this->items) >= $limit) { + return; + } + + } + + } + + private function extractHotTopics($html) { + + $topics = $html->find('#popbox ul li') + or returnServerError('Could not find topics!'); + + $limit = $this->getInput('limit') ?: -1; + + foreach($topics as $topic) { + + $anchor = $topic->find('a', 0) + or returnServerError('Could not find anchor!'); + + $item = array(); + + $item['uri'] = $this->getTarget($anchor); + $item['title'] = $anchor->title; + + $this->items[] = $item; + + if($limit > 0 && count($this->items) >= $limit) { + return; + } + + } + + } + + private function extractCustomFeed($html) { + + $boxes = $html->find('#boxx .boxes') + or returnServerError('Could not find boxes!'); + + foreach($boxes as $box) { + + $anchor = $box->find('span.boxtitles a', 0) + or returnServerError('Could not find box anchor!'); + + $author = '<a href="' . $anchor->href . '">' . trim($anchor->plaintext) . '</a>'; + $uri = $anchor->href; + + $box_html = getSimpleHTMLDOM($uri) + or returnServerError('Could not load custom feed!'); + + $this->extractFeed($box_html, $author); + + } + + } + + private function getTarget($anchor) { + + // Anchors are linked to Skimfeed, luckily the target URI is encoded + // in that URI via '&u=<URI>': + $query = parse_url($anchor->href, PHP_URL_QUERY); + + foreach(explode('&', $query) as $parameter) { + + list($key, $value) = explode('=', $parameter); + + if($key !== 'u') { + continue; + } + + return urldecode($value); + + } + + } + + /** + * dev-mode! + * Requires '&format=Html' + * + * Returns the 'box' array from the source site + */ + private function exportBoxChannels() { + $html = getSimpleHTMLDOMCached(static::URI) + or returnServerError('No contents received from Skimfeed!'); + + if(!$this->isCompatible($html)) { + returnServerError('Skimfeed version is not compatible!'); + } + + $boxes = $html->find('#boxx .boxes') + or returnServerError('Could not find boxes!'); + + // begin of 'channel' list + $message = <<<EOD +'box_channel' => array( + 'name' => 'Channel', + 'type' => 'list', + 'required' => true, + 'title' => 'Select your channel', + 'values' => array( + +EOD; + + foreach($boxes as $box) { + + $anchor = $box->find('span.boxtitles a', 0) + or returnServerError('Could not find box anchor!'); + + $title = trim($anchor->plaintext); + $uri = $anchor->href; + + // add value + $message .= "\t\t'{$title}' => '{$uri}', \n"; + + } + + // end of 'box' list + $message .= <<<EOD + ) +), +EOD; + + echo <<<EOD +<!DOCTYPE html> + +<html> + <body> + <code style="white-space: pre-wrap;">{$message}</code> + </body> +</html> +EOD; + + } + + /** + * dev-mode! + * Requires '&format=Html' + * + * Returns the 'techs' array from the source site + */ + private function exportTechChannels() { + $html = getSimpleHTMLDOMCached(static::URI) + or returnServerError('No contents received from Skimfeed!'); + + if(!$this->isCompatible($html)) { + returnServerError('Skimfeed version is not compatible!'); + } + + $channels = $html->find('#menubar a') + or returnServerError('Could not find channels!'); + + // begin of 'tech_channel' list + $message = <<<EOD +'tech_channel' => array( + 'name' => 'Tech channel', + 'type' => 'list', + 'required' => true, + 'title' => 'Select your tech channel', + 'values' => array( + +EOD; + + foreach($channels as $channel) { + + if($channel->href === '#' + || $channel->class === 'homelink' + || $channel->plaintext === 'Twitter' + || $channel->plaintext === 'Weather' + || $channel->plaintext === '+Custom') { + continue; + } + + $title = trim($channel->plaintext); + $uri = '/' . $channel->href; + + $message .= "\t\t'{$title}' => array(\n"; + + $channel_html = getSimpleHTMLDOMCached(static::URI . $uri) + or returnServerError('Could not load tech channel ' . $channel->plaintext . '!'); + + $boxes = $channel_html->find('#boxx .boxes') + or returnServerError('Could not find boxes!'); + + foreach($boxes as $box) { + + $anchor = $box->find('span.boxtitles a', 0) + or returnServerError('Could not find box anchor!'); + + $boxtitle = trim($anchor->plaintext); + $boxuri = $anchor->href; + + $message .= "\t\t\t'{$boxtitle}' => '{$boxuri}', \n"; + + } + + $message .= "\t\t),\n"; + + } + + // end of 'box' list + $message .= <<<EOD + ) +), +EOD; + + echo <<<EOD +<!DOCTYPE html> + +<html> + <body> + <code style="white-space: pre-wrap;">{$message}</code> + </body> +</html> +EOD; + } + + + /** + * Checks if the reported skimfeed version is compatible + */ + private function isCompatible($html) { + $title = $html->find('title', 0); + + if(!$title) { + return false; + } + + if($title->plaintext === 'Skimfeed V5.5 - Tech News') { + return true; + } + + return false; + } + +} diff --git a/bridges/SupInfoBridge.php b/bridges/SupInfoBridge.php index 40160e4..697aadf 100644 --- a/bridges/SupInfoBridge.php +++ b/bridges/SupInfoBridge.php @@ -13,6 +13,10 @@ class SupInfoBridge extends BridgeAbstract { ) )); + public function getIcon() { + return self::URI . '/favicon.png'; + } + public function collectData() { if(empty($this->getInput('tag'))) { @@ -31,7 +35,7 @@ class SupInfoBridge extends BridgeAbstract { } } - public function fetchArticle($link) { + private function fetchArticle($link) { $articleHTML = getSimpleHTMLDOM(self::URI . $link) or returnServerError('Unable to fetch article !'); diff --git a/bridges/SuperSmashBlogBridge.php b/bridges/SuperSmashBlogBridge.php index 9216ef6..a2ce47d 100644 --- a/bridges/SuperSmashBlogBridge.php +++ b/bridges/SuperSmashBlogBridge.php @@ -25,7 +25,7 @@ class SuperSmashBlogBridge extends BridgeAbstract { $video = $article['acf']['link_url']; if (strlen($video) != 0) { - $video = str_get_html('<a href="' . $video .'">Youtube video</a>'); + $video = str_get_html('<a href="' . $video . '">Youtube video</a>'); } else { $video = ''; } diff --git a/bridges/TagBoardBridge.php b/bridges/TagBoardBridge.php index b79847e..2a2f51c 100644 --- a/bridges/TagBoardBridge.php +++ b/bridges/TagBoardBridge.php @@ -14,6 +14,10 @@ class TagBoardBridge extends BridgeAbstract { ) )); + public function getIcon() { + return 'https://static.tagboard.com/public/favicon-32x32.png'; + } + public function collectData(){ $link = 'https://post-cache.tagboard.com/search/' . $this->getInput('u'); diff --git a/bridges/TebeoBridge.php b/bridges/TebeoBridge.php index 9050439..083ea94 100644 --- a/bridges/TebeoBridge.php +++ b/bridges/TebeoBridge.php @@ -21,6 +21,10 @@ class TebeoBridge extends FeedExpander { ) )); + public function getIcon() { + return self::URI . 'images/header_logo.png'; + } + public function collectData(){ $url = self::URI . '/le-replay/' . $this->getInput('cat'); $html = getSimpleHTMLDOM($url) @@ -31,7 +35,7 @@ class TebeoBridge extends FeedExpander { $item['uri'] = $element->find('a', 0)->href; $item['title'] = $element->find('h3', 0)->plaintext; $item['timestamp'] = strtotime($element->find('p.moment-format-day', 0)->plaintext); - $item['content'] = '<a href="'.$item['uri'].'"><img alt="" src="'.$element->find('img', 0)->src.'"></a>'; + $item['content'] = '<a href="' . $item['uri'] . '"><img alt="" src="' . $element->find('img', 0)->src . '"></a>'; $this->items[] = $item; } } diff --git a/bridges/TheHackerNewsBridge.php b/bridges/TheHackerNewsBridge.php index 4106658..d0d2e97 100644 --- a/bridges/TheHackerNewsBridge.php +++ b/bridges/TheHackerNewsBridge.php @@ -8,67 +8,66 @@ class TheHackerNewsBridge extends BridgeAbstract { public function collectData(){ - function stripWithDelimiters($string, $start, $end){ - while(strpos($string, $start) !== false) { - $section_to_remove = substr($string, strpos($string, $start)); - $section_to_remove = substr($section_to_remove, 0, strpos($section_to_remove, $end) + strlen($end)); - $string = str_replace($section_to_remove, '', $string); - } - - return $string; - } - - function stripRecursiveHtmlSection($string, $tag_name, $tag_start){ - $open_tag = '<' . $tag_name; - $close_tag = '</' . $tag_name . '>'; - $close_tag_length = strlen($close_tag); - if(strpos($tag_start, $open_tag) === 0) { - while(strpos($string, $tag_start) !== false) { - $max_recursion = 100; - $section_to_remove = null; - $section_start = strpos($string, $tag_start); - $search_offset = $section_start; - do { - $max_recursion--; - $section_end = strpos($string, $close_tag, $search_offset); - $search_offset = $section_end + $close_tag_length; - $section_to_remove = substr( - $string, - $section_start, - $section_end - $section_start + $close_tag_length - ); - - $open_tag_count = substr_count($section_to_remove, $open_tag); - $close_tag_count = substr_count($section_to_remove, $close_tag); - } while($open_tag_count > $close_tag_count && $max_recursion > 0); - $string = str_replace($section_to_remove, '', $string); - } - } - return $string; - } - $html = getSimpleHTMLDOM($this->getURI()) or returnServerError('Could not request TheHackerNews: ' . $this->getURI()); $limit = 0; - foreach($html->find('article') as $element) { + foreach($html->find('div.body-post') as $element) { if($limit < 5) { - $article_url = $element->find('a.entry-title', 0)->href; - $article_author = trim($element->find('span.vcard', 0)->plaintext); - $article_title = $element->find('a.entry-title', 0)->plaintext; - $article_timestamp = strtotime($element->find('span.updated', 0)->plaintext); - $article = getSimpleHTMLDOM($article_url) - or returnServerError('Could not request TheHackerNews: ' . $article_url); + $article_url = $element->find('a.story-link', 0)->href; + $article_author = trim($element->find('i.fa-user', 0)->parent()->plaintext); + $article_title = $element->find('h2.home-title', 0)->plaintext; + + //Date without time + $article_timestamp = strtotime( + extractFromDelimiters( + $element->find('i.fa-calendar', 0)->parent()->outertext, + '</i>', + '<span>' + ) + ); + + //Article thumbnail in lazy-loading image + if (is_object($element->find('img[data-echo]', 0))) { + $article_thumbnail = array( + extractFromDelimiters( + $element->find('img[data-echo]', 0)->outertext, + "data-echo='", + "'" + ) + ); + } else { + $article_thumbnail = array(); + } + + if ($article = getSimpleHTMLDOMCached($article_url)) { + + //Article body + $contents = $article->find('div.articlebody', 0)->innertext; + $contents = stripRecursiveHtmlSection($contents, 'div', '<div class="ad_'); + $contents = stripWithDelimiters($contents, 'id="google_ads', '</iframe>'); + $contents = stripWithDelimiters($contents, '<script', '</script>'); - $contents = $article->find('div.articlebodyonly', 0)->innertext; - $contents = stripRecursiveHtmlSection($contents, 'div', '<div class=\'clear\''); - $contents = stripWithDelimiters($contents, '<script', '</script>'); + //Date with time + if (is_object($article->find('meta[itemprop=dateModified]', 0))) { + $article_timestamp = strtotime( + extractFromDelimiters( + $article->find('meta[itemprop=dateModified]', 0)->outertext, + "content='", + "'" + ) + ); + } + } else { + $contents = 'Could not request TheHackerNews: ' . $article_url; + } $item = array(); $item['uri'] = $article_url; $item['title'] = $article_title; $item['author'] = $article_author; + $item['enclosures'] = $article_thumbnail; $item['timestamp'] = $article_timestamp; $item['content'] = trim($contents); $this->items[] = $item; diff --git a/bridges/ThePirateBayBridge.php b/bridges/ThePirateBayBridge.php index 0deaded..9aefcbb 100644 --- a/bridges/ThePirateBayBridge.php +++ b/bridges/ThePirateBayBridge.php @@ -3,7 +3,7 @@ class ThePirateBayBridge extends BridgeAbstract { const MAINTAINER = 'mitsukarenai'; const NAME = 'The Pirate Bay'; - const URI = 'https://thepiratebay.org/'; + const URI = 'https://thepiratebay.wf/'; const DESCRIPTION = 'Returns results for the keywords. You can put several list of keywords by separating them with a semicolon (e.g. "one show;another show"). Category based search needs the category number as input. User based diff --git a/bridges/TheTVDBBridge.php b/bridges/TheTVDBBridge.php index 63af1ea..38b45a8 100644 --- a/bridges/TheTVDBBridge.php +++ b/bridges/TheTVDBBridge.php @@ -158,6 +158,10 @@ class TheTVDBBridge extends BridgeAbstract { } } + public function getIcon() { + return self::URI . 'application/themes/thetvdb/images/logo.png'; + } + public function collectData(){ $serie_id = $this->getInput('serie_id'); $nbepisode = $this->getInput('nb_episode'); diff --git a/bridges/TheYeteeBridge.php b/bridges/TheYeteeBridge.php new file mode 100644 index 0000000..fa5a645 --- /dev/null +++ b/bridges/TheYeteeBridge.php @@ -0,0 +1,41 @@ +<?php +class TheYeteeBridge extends BridgeAbstract { + + const MAINTAINER = 'Monsieur Poutounours'; + const NAME = 'TheYetee'; + const URI = 'https://theyetee.com'; + const CACHE_TIMEOUT = 14400; // 4 h + const DESCRIPTION = 'Fetch daily shirts from The Yetee'; + + public function collectData(){ + + $html = getSimpleHTMLDOM(self::URI) + or returnServerError('Could not request The Yetee.'); + + $div = $html->find('.hero-col'); + foreach($div as $element) { + + $item = array(); + $item['enclosures'] = array(); + + $title = $element->find('h2', 0)->plaintext; + $item['title'] = $title; + + $author = trim($element->find('div[class=credit]', 0)->plaintext); + $item['author'] = $author; + + $uri = $element->find('div[class=controls] a', 0)->href; + $item['uri'] = static::URI . $uri; + + $content = '<p>' . $element->find('section[class=product-listing-info] p', -1)->plaintext . '</p>'; + $photos = $element->find('a[class=js-modaal-gallery] img'); + foreach($photos as $photo) { + $content = $content . "<br /><img src='$photo->src' />"; + $item['enclosures'][] = $photo->src; + } + $item['content'] = $content; + + $this->items[] = $item; + } + } +} diff --git a/bridges/ThingiverseBridge.php b/bridges/ThingiverseBridge.php new file mode 100644 index 0000000..2e10881 --- /dev/null +++ b/bridges/ThingiverseBridge.php @@ -0,0 +1,167 @@ +<?php +class ThingiverseBridge extends BridgeAbstract { + + const NAME = 'Thingiverse Search'; + const URI = 'https://thingiverse.com'; + const DESCRIPTION = 'Returns feeds for search results'; + const MAINTAINER = 'AntoineTurmel'; + const PARAMETERS = array( + array( + 'query' => array( + 'name' => 'Search query', + 'type' => 'text', + 'required' => true, + 'title' => 'Insert your search term here', + 'exampleValue' => 'Enter your search term' + ), + 'sortby' => array( + 'name' => 'Sort by', + 'type' => 'list', + 'required' => false, + 'values' => array( + 'Relevant' => 'relevant', + 'Text' => 'text', + 'Popular' => 'popular', + '# of Makes' => 'makes', + 'Newest' => 'newest', + ), + 'defaultValue' => 'newest' + ), + 'category' => array( + 'name' => 'Category', + 'type' => 'list', + 'required' => false, + 'values' => array( + 'Any' => '', + '3D Printing' => '73', + 'Art' => '63', + 'Fashion' => '64', + 'Gadgets' => '65', + 'Hobby' => '66', + 'Household' => '67', + 'Learning' => '69', + 'Models' => '70', + 'Tools' => '71', + 'Toys & Games' => '72', + '2D Art' => '144', + 'Art Tools' => '75', + 'Coins & Badges' => '143', + 'Interactive Art' => '78', + 'Math Art' => '79', + 'Scans & Replicas' => '145', + 'Sculptures' => '80', + 'Signs & Logos' => '76', + 'Accessories' => '81', + 'Bracelets' => '82', + 'Costume' => '142', + 'Earrings' => '139', + 'Glasses' => '83', + 'Jewelry' => '84', + 'Keychains' => '130', + 'Rings' => '85', + 'Audio' => '141', + 'Camera' => '86', + 'Computer' => '87', + 'Mobile Phone' => '88', + 'Tablet' => '90', + 'Video Games' => '91', + 'Automotive' => '155', + 'DIY' => '93', + 'Electronics' => '92', + 'Music' => '94', + 'R/C Vehicles' => '95', + 'Robotics' => '96', + 'Sport & Outdoors' => '140', + 'Bathroom' => '147', + 'Containers' => '146', + 'Decor' => '97', + 'Household Supplies' => '99', + 'Kitchen & Dining' => '100', + 'Office' => '101', + 'Organization' => '102', + 'Outdoor & Garden' => '98', + 'Pets' => '103', + 'Replacement Parts' => '153', + 'Biology' => '106', + 'Engineering' => '104', + 'Math' => '105', + 'Physics & Astronomy' => '148', + 'Animals' => '107', + 'Buildings & Structures' => '108', + 'Creatures' => '109', + 'Food & Drink' => '110', + 'Model Furniture' => '111', + 'Model Robots' => '115', + 'People' => '112', + 'Props' => '114', + 'Vehicles' => '116', + 'Hand Tools' => '118', + 'Machine Tools' => '117', + 'Parts' => '119', + 'Tool Holders & Boxes' => '120', + 'Chess' => '151', + 'Construction Toys' => '121', + 'Dice' => '122', + 'Games' => '123', + 'Mechanical Toys' => '124', + 'Playsets' => '113', + 'Puzzles' => '125', + 'Toy & Game Accessories' => '149', + '3D Printer Accessories' => '127', + '3D Printer Extruders' => '152', + '3D Printer Parts' => '128', + '3D Printers' => '126', + '3D Printing Tests' => '129', + ), + 'defaultValue' => '' + ), + 'showimage' => array( + 'name' => 'Show image in content', + 'type' => 'checkbox', + 'required' => false, + 'title' => 'Activate to show the image in the content', + 'defaultValue' => 'checked' + ) + ) + ); + + public function collectData(){ + $html = getSimpleHTMLDOM($this->getURI()) + or returnServerError('Failed to receive ' . $this->getURI()); + + $results = $html->find('div.thing-card'); + + foreach($results as $result) { + + $item = array(); + + $item['title'] = $result->find('span.ellipsis', 0); + $item['uri'] = self::URI . $result->find('a', 1)->href; + $item['author'] = $result->find('span.item-creator', 0); + $item['content'] = ''; + + $image = $result->find('img.card-img', 0)->src; + + if($this->getInput('showimage')) { + $item['content'] .= '<img src="' . $image . '">'; + } + + $item['enclosures'] = array($image); + + $this->items[] = $item; + } + } + + public function getURI(){ + if(!is_null($this->getInput('query'))) { + $uri = self::URI . '/search?q=' . urlencode($this->getInput('query')); + $uri .= '&sort=' . $this->getInput('sortby'); + $uri .= '&category_id=' . $this->getInput('category'); + + return $uri; + } + + return parent::getURI(); + } + +} diff --git a/bridges/Torrent9Bridge.php b/bridges/Torrent9Bridge.php deleted file mode 100644 index 40db4ac..0000000 --- a/bridges/Torrent9Bridge.php +++ /dev/null @@ -1,102 +0,0 @@ -<?php -class Torrent9Bridge extends BridgeAbstract { - - const MAINTAINER = 'lagaisse'; - const NAME = 'Torrent9 Bridge'; - const URI = 'http://www.torrent9.pe'; - const CACHE_TIMEOUT = 86400; // 24h = 86400s - const DESCRIPTION = 'Returns latest torrents'; - - const PAGE_SERIES = 'torrents_series'; - const PAGE_SERIES_VOSTFR = 'torrents_series_vostfr'; - const PAGE_SERIES_FR = 'torrents_series_french'; - - const PARAMETERS = array( - 'From search' => array( - 'q' => array( - 'name' => 'Search', - 'required' => true, - 'title' => 'Type your search' - ) - ), - 'By page' => array( - 'page' => array( - 'name' => 'Page', - 'type' => 'list', - 'required' => false, - 'values' => array( - 'Series' => self::PAGE_SERIES, - 'Series VOST' => self::PAGE_SERIES_VOSTFR, - 'Series FR' => self::PAGE_SERIES_FR, - ), - 'defaultValue' => self::PAGE_SERIES - ) - ) - ); - - public function collectData(){ - - if($this->queriedContext === 'From search') { - $request = str_replace(' ', '-', trim($this->getInput('q'))); - $page = self::URI . '/search_torrent/' . urlencode($request) . '.html'; - } else { - $request = $this->getInput('page'); - $page = self::URI . '/' . $request . '.html'; - } - - $html = getSimpleHTMLDOM($page) - or returnServerError('No results for this query.'); - - foreach($html->find('table', 0)->find('tr') as $episode) { - if($episode->parent->tag == 'tbody') { - - $urlepisode = self::URI . $episode->find('a', 0)->getAttribute('href'); - - //30 years = forever - $htmlepisode = getSimpleHTMLDOMCached($urlepisode, 86400 * 366 * 30); - - $item = array(); - $item['author'] = $episode->find('a', 0)->text(); - $item['title'] = $episode->find('a', 0)->text(); - $item['id'] = $episode->find('a', 0)->getAttribute('href'); - $item['pubdate'] = $this->getCachedDate($urlepisode); - - $textefiche = $htmlepisode->find('.movie-information', 0)->find('p', 1); - if(isset($textefiche)) { - $item['content'] = $textefiche->text(); - } else { - $p = $htmlepisode->find('.movie-information', 0)->find('p'); - if(!empty($p)) { - $item['content'] = $htmlepisode->find('.movie-information', 0)->find('p', 0)->text(); - } - } - - $item['id'] = $episode->find('a', 0)->getAttribute('href'); - $item['uri'] = self::URI . $htmlepisode->find('.download', 0)->getAttribute('href'); - - $this->items[] = $item; - } - } - } - - - public function getName(){ - if(!is_null($this->getInput('q'))) { - return $this->getInput('q') . ' : ' . self::NAME; - } - - return parent::getName(); - } - - private function getCachedDate($url){ - debugMessage('getting pubdate from url ' . $url . ''); - // Initialize cache - $cache = Cache::create('FileCache'); - $cache->setPath(CACHE_DIR . '/pages'); - $params = [$url]; - $cache->setParameters($params); - // Get cachefile timestamp - $time = $cache->getTime(); - return ($time !== false ? $time : time()); - } -} diff --git a/bridges/UnsplashBridge.php b/bridges/UnsplashBridge.php index ee1040a..ae76734 100644 --- a/bridges/UnsplashBridge.php +++ b/bridges/UnsplashBridge.php @@ -55,7 +55,7 @@ class UnsplashBridge extends BridgeAbstract { $item['uri'] = str_replace( array('q=75', 'w=400'), array("q=$quality", "w=$width"), - $thumbnail->src).'.jpg'; // '.jpg' only for format hint + $thumbnail->src) . '.jpg'; // '.jpg' only for format hint $item['timestamp'] = time(); $item['title'] = $thumbnail->alt; diff --git a/bridges/VkBridge.php b/bridges/VkBridge.php index 70c0db4..d4e84d9 100644 --- a/bridges/VkBridge.php +++ b/bridges/VkBridge.php @@ -17,8 +17,14 @@ class VkBridge extends BridgeAbstract ) ); + protected $videos = array(); protected $pageName; + protected function getAccessToken() + { + return 'c8071613517c155c6cfbd2a059b2718e9c37b89094c4766834969dda75f657a2c1cbb49bab4c5e649f1db'; + } + public function getURI() { if (!is_null($this->getInput('u'))) { @@ -51,11 +57,20 @@ class VkBridge extends BridgeAbstract $pageName = $pageName->plaintext; $this->pageName = htmlspecialchars_decode($pageName); } + foreach ($html->find('div.replies') as $comment_block) { + $comment_block->outertext = ''; + } + $html->load($html->save()); + $pinned_post_item = null; $last_post_id = 0; foreach ($html->find('.post') as $post) { + defaultLinkTo($post, self::URI); + + $post_videos = array(); + $is_pinned_post = false; if (strpos($post->getAttribute('class'), 'post_fixed') !== false) { $is_pinned_post = true; @@ -114,7 +129,7 @@ class VkBridge extends BridgeAbstract } $article_title = $article->find($article_title_selector, 0)->innertext; $article_author = $article->find($article_author_selector, 0)->innertext; - $article_link = self::URI . ltrim($article->getAttribute('href'), '/'); + $article_link = $article->getAttribute('href'); $article_img_element_style = $article->find($article_thumb_selector, 0)->getAttribute('style'); preg_match('/background-image: url\((.*)\)/', $article_img_element_style, $matches); if (count($matches) > 0) { @@ -126,20 +141,22 @@ class VkBridge extends BridgeAbstract // get video on post $video = $post->find('div.post_video_desc', 0); + $main_video_link = ''; if (is_object($video)) { $video_title = $video->find('div.post_video_title', 0)->plaintext; - $video_link = self::URI . ltrim( $video->find('a.lnk', 0)->getAttribute('href'), '/' ); - $content_suffix .= "<br>Video: <a href='$video_link'>$video_title</a>"; + $video_link = $video->find('a.lnk', 0)->getAttribute('href'); + $this->appendVideo($video_title, $video_link, $content_suffix, $post_videos); $video->outertext = ''; + $main_video_link = $video_link; } // get all other videos foreach($post->find('a.page_post_thumb_video') as $a) { - $video_title = $a->getAttribute('aria-label'); + $video_title = htmlspecialchars_decode($a->getAttribute('aria-label')); $temp = explode(' ', $video_title, 2); if (count($temp) > 1) $video_title = $temp[1]; - $video_link = self::URI . ltrim( $a->getAttribute('href'), '/' ); - $content_suffix .= "<br>Video: <a href='$video_link'>$video_title</a>"; + $video_link = $a->getAttribute('href'); + if ($video_link != $main_video_link) $this->appendVideo($video_title, $video_link, $content_suffix, $post_videos); $a->outertext = ''; } @@ -155,14 +172,14 @@ class VkBridge extends BridgeAbstract foreach($post->find('.page_album_wrap') as $el) { $a = $el->find('.page_album_link', 0); $album_title = $a->find('.page_album_title_text', 0)->getAttribute('title'); - $album_link = self::URI . ltrim($a->getAttribute('href'), '/'); + $album_link = $a->getAttribute('href'); $el->outertext = ''; $content_suffix .= "<br>Album: <a href='$album_link'>$album_title</a>"; } // get photo documents foreach($post->find('a.page_doc_photo_href') as $a) { - $doc_link = self::URI . ltrim($a->getAttribute('href'), '/'); + $doc_link = $a->getAttribute('href'); $doc_gif_label_element = $a->find('.page_gif_label', 0); $doc_title_element = $a->find('.doc_label', 0); @@ -188,7 +205,7 @@ class VkBridge extends BridgeAbstract if (is_object($doc_title_element)) { $doc_title = $doc_title_element->innertext; - $doc_link = self::URI . ltrim($doc_title_element->getAttribute('href'), '/'); + $doc_link = $doc_title_element->getAttribute('href'); $content_suffix .= "<br>Doc: <a href='$doc_link'>$doc_title</a>"; } else { @@ -228,20 +245,29 @@ class VkBridge extends BridgeAbstract $item = array(); $item['content'] = strip_tags(backgroundToImg($post->find('div.wall_text', 0)->innertext), '<br><img>'); $item['content'] .= $content_suffix; + $item['categories'] = array(); + + // get post hashtags + foreach($post->find('a') as $a) { + $href = $a->getAttribute('href'); + $prefix = '/feed?section=search&q=%23'; + $innertext = $a->innertext; + if ($href && substr($href, 0, strlen($prefix)) === $prefix) { + $item['categories'][] = urldecode(substr($href, strlen($prefix))); + } else if (substr($innertext, 0, 1) == '#') { + $item['categories'][] = $innertext; + } + } // get post link $post_link = $post->find('a.post_link', 0)->getAttribute('href'); preg_match('/wall-?\d+_(\d+)/', $post_link, $preg_match_result); $item['post_id'] = intval($preg_match_result[1]); - if (substr(self::URI, -1) == '/') { - $post_link = self::URI . ltrim($post_link, '/'); - } else { - $post_link = self::URI . $post_link; - } $item['uri'] = $post_link; $item['timestamp'] = $this->getTime($post); $item['title'] = $this->getTitle($item['content']); $item['author'] = $post_author; + $item['videos'] = $post_videos; if ($is_pinned_post) { // do not append it now $pinned_post_item = $item; @@ -252,16 +278,18 @@ class VkBridge extends BridgeAbstract } - if (is_null($pinned_post_item)) { - return; - } else if (count($this->items) == 0) { - $this->items[] = $pinned_post_item; - } else if ($last_post_id < $pinned_post_item['post_id']) { - $this->items[] = $pinned_post_item; - usort($this->items, function ($item1, $item2) { - return $item2['post_id'] - $item1['post_id']; - }); + if (!is_null($pinned_post_item)) { + if (count($this->items) == 0) { + $this->items[] = $pinned_post_item; + } else if ($last_post_id < $pinned_post_item['post_id']) { + $this->items[] = $pinned_post_item; + usort($this->items, function ($item1, $item2) { + return $item2['post_id'] - $item1['post_id']; + }); + } } + + $this->getCleanVideoLinks(); } private function getPhoto($a) { @@ -326,7 +354,7 @@ class VkBridge extends BridgeAbstract } - public function getContents() + private function getContents() { ini_set('user-agent', 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:53.0) Gecko/20100101 Firefox/53.0'); @@ -335,5 +363,51 @@ class VkBridge extends BridgeAbstract return getContents($this->getURI(), $header); } + protected function appendVideo($video_title, $video_link, &$content_suffix, array &$post_videos) + { + if (!$video_title) $video_title = '(empty)'; + + preg_match('/video([0-9-]+_[0-9]+)/', $video_link, $preg_match_result); + + if (count($preg_match_result) > 1) { + $video_id = $preg_match_result[1]; + $this->videos[ $video_id ] = array( + 'url' => $video_link, + 'title' => $video_title, + ); + $post_videos[] = $video_id; + } else { + $content_suffix .= '<br>Video: <a href="' . htmlspecialchars($video_link) . '">' . $video_title . '</a>'; + } + } + + protected function getCleanVideoLinks() { + $result = $this->api('video.get', array( + 'videos' => implode(',', array_keys($this->videos)), + 'count' => 200 + )); + if (isset($result['error'])) return; + + foreach($result['response']['items'] as $item) { + $video_id = strval($item['owner_id']) . '_' . strval($item['id']); + $this->videos[$video_id]['url'] = $item['player']; + } + + foreach($this->items as &$item) { + foreach($item['videos'] as $video_id) { + $video_link = $this->videos[$video_id]['url']; + $video_title = $this->videos[$video_id]['title']; + $item['content'] .= '<br>Video: <a href="' . htmlspecialchars($video_link) . '">' . $video_title . '</a>'; + } + unset($item['videos']); + } + } + + protected function api($method, array $params) + { + $params['v'] = '5.80'; + $params['access_token'] = $this->getAccessToken(); + return json_decode( getContents('https://api.vk.com/method/' . $method . '?' . http_build_query($params)), true ); + } } diff --git a/bridges/WeLiveSecurityBridge.php b/bridges/WeLiveSecurityBridge.php index 466a4b2..59a094a 100644 --- a/bridges/WeLiveSecurityBridge.php +++ b/bridges/WeLiveSecurityBridge.php @@ -3,37 +3,24 @@ class WeLiveSecurityBridge extends FeedExpander { const MAINTAINER = 'ORelio'; const NAME = 'We Live Security'; - const URI = 'http://www.welivesecurity.com/'; + const URI = 'https://www.welivesecurity.com/'; const DESCRIPTION = 'Returns the newest articles.'; - private function stripWithDelimiters($string, $start, $end){ - while(strpos($string, $start) !== false) { - $section_to_remove = substr($string, strpos($string, $start)); - $section_to_remove = substr($section_to_remove, 0, strpos($section_to_remove, $end) + strlen($end)); - $string = str_replace($section_to_remove, '', $string); - } - - return $string; - } - - protected function parseItem($item){ $item = parent::parseItem($item); $article_html = getSimpleHTMLDOMCached($item['uri']); if(!$article_html) { - $item['content'] .= '<p>Could not request ' . $this->getName() . ': ' . $item['uri'] . '</p>'; + $item['content'] .= '<p><em>Could not request ' . $this->getName() . ': ' . $item['uri'] . '</em></p>'; return $item; } - $article_content = $article_html->find('div.wlistingsingletext', 0)->innertext; - $article_content = $this->stripWithDelimiters($article_content, '<script', '</script>'); - $article_content = '<p><b>' - . $item['content'] - . '</b></p>' - . trim($article_content); - - $item['content'] = $article_content; + $article_content = $article_html->find('div.formatted', 0)->innertext; + $article_content = stripWithDelimiters($article_content, '<script', '</script>'); + $article_content = stripRecursiveHTMLSection($article_content, 'div', '<div class="comments'); + $article_content = stripRecursiveHTMLSection($article_content, 'div', '<div class="similar-articles'); + $article_content = stripRecursiveHTMLSection($article_content, 'span', '<span class="meta'); + $item['content'] = trim($article_content); return $item; } diff --git a/bridges/WhydBridge.php b/bridges/WhydBridge.php index 347db6e..d14c22a 100644 --- a/bridges/WhydBridge.php +++ b/bridges/WhydBridge.php @@ -16,6 +16,11 @@ class WhydBridge extends BridgeAbstract { private $userName = ''; + public function getIcon() { + return self::URI . 'assets/favicons/ +32-6b62a9f14d5e1a9213090d8f00f286bba3a6022381a76390d1d0926493b12593.png?v=6'; + } + public function collectData(){ $html = ''; if(strlen(preg_replace('/[^0-9a-f]/', '', $this->getInput('u'))) == 24) { diff --git a/bridges/WordPressBridge.php b/bridges/WordPressBridge.php index b367adc..2a750a9 100644 --- a/bridges/WordPressBridge.php +++ b/bridges/WordPressBridge.php @@ -3,8 +3,7 @@ class WordPressBridge extends FeedExpander { const MAINTAINER = 'aledeg'; const NAME = 'Wordpress Bridge'; const URI = 'https://wordpress.org/'; - const CACHE_TIMEOUT = 10800; // 3h - const DESCRIPTION = 'Returns the newest full posts of a Wordpress powered website'; + const DESCRIPTION = 'Returns the newest full posts of a WordPress powered website'; const PARAMETERS = array( array( 'url' => array( @@ -13,8 +12,8 @@ class WordPressBridge extends FeedExpander { ) )); - private function clearContent($content){ - $content = preg_replace('/<script[^>]*>[^<]*<\/script>/', '', $content); + private function cleanContent($content){ + $content = stripWithDelimiters($content, '<script', '</script>'); $content = preg_replace('/<div class="wpa".*/', '', $content); $content = preg_replace('/<form.*\/form>/', '', $content); return $content; @@ -27,6 +26,10 @@ class WordPressBridge extends FeedExpander { $article = null; switch(true) { + case !is_null($article_html->find('[itemprop=articleBody]', 0)): + // highest priority content div + $article = $article_html->find('[itemprop=articleBody]', 0); + break; case !is_null($article_html->find('article', 0)): // most common content div $article = $article_html->find('article', 0); @@ -39,15 +42,37 @@ class WordPressBridge extends FeedExpander { // another common content div $article = $article_html->find('.post-content', 0); break; - case !is_null($article_html->find('.post', 0)): // for old WordPress themes without HTML5 $article = $article_html->find('.post', 0); break; } + foreach ($article->find('h1.entry-title') as $title) + if ($title->plaintext == $item['title']) + $title->outertext = ''; + + $article_image = $article_html->find('img.wp-post-image', 0); + if(!empty($item['content']) && (!is_object($article_image) || empty($article_image->src))) { + $article_image = str_get_html($item['content'])->find('img.wp-post-image', 0); + } + if(is_object($article_image) && !empty($article_image->src)) { + if(empty($article_image->getAttribute('data-lazy-src'))) { + $article_image = $article_image->src; + } else { + $article_image = $article_image->getAttribute('data-lazy-src'); + } + $mime_type = getMimeType($article_image); + if (strpos($mime_type, 'image') === false) + $article_image .= '#.image'; // force image + if (empty($item['enclosures'])) + $item['enclosures'] = array($article_image); + else + $item['enclosures'] = array_merge($item['enclosures'], $article_image); + } + if(!is_null($article)) { - $item['content'] = $this->clearContent($article->innertext); + $item['content'] = $this->cleanContent($article->innertext); } return $item; diff --git a/bridges/WordPressPluginUpdateBridge.php b/bridges/WordPressPluginUpdateBridge.php index cb57df8..fb4a57e 100644 --- a/bridges/WordPressPluginUpdateBridge.php +++ b/bridges/WordPressPluginUpdateBridge.php @@ -77,7 +77,7 @@ class WordPressPluginUpdateBridge extends BridgeAbstract { debugMessage('getting pubdate from url ' . $url . ''); // Initialize cache $cache = Cache::create('FileCache'); - $cache->setPath(CACHE_DIR . '/pages'); + $cache->setPath(PATH_CACHE . '/pages'); $params = [$url]; $cache->setParameters($params); // Get cachefile timestamp diff --git a/bridges/XenForoBridge.php b/bridges/XenForoBridge.php new file mode 100644 index 0000000..75c0f6d --- /dev/null +++ b/bridges/XenForoBridge.php @@ -0,0 +1,462 @@ +<?php +/** + * This bridge generates feeds for threads from forums running XenForo version 2 + * + * Examples: + * - https://xenforo.com/community/ + * - http://www.ign.com/boards/ + * + * Notice: XenForo does provide RSS feeds for forums. For example: + * - https://xenforo.com/community/forums/-/index.rss + * + * For more information on XenForo, visit + * - https://xenforo.com/ + * - https://en.wikipedia.org/wiki/XenForo + */ +class XenForoBridge extends BridgeAbstract { + + // Bridge specific constants + const CONTEXT_THREAD = 'Thread'; + const XENFORO_VERSION_1 = '1.0'; + const XENFORO_VERSION_2 = '2.0'; + + // RSS-Bridge constants + const NAME = 'XenForo Bridge'; + const URI = 'https://xenforo.com/'; + const DESCRIPTION = 'Generates feeds for threads in forums powered by XenForo'; + const MAINTAINER = 'logmanoriginal'; + const PARAMETERS = array( + self::CONTEXT_THREAD => array( + 'url' => array( + 'name' => 'Thread URL', + 'type' => 'text', + 'required' => true, + 'title' => 'Insert URL to the thread for which the feed should be generated', + 'exampleValue' => 'https://xenforo.com/community/threads/guide-to-suggestions.2285/' + ) + ), + 'global' => array( + 'limit' => array( + 'name' => 'Limit', + 'type' => 'number', + 'required' => false, + 'title' => 'Specify maximum number of elements to return in the feed', + 'defaultValue' => 10 + ) + ) + ); + const CACHE_TIMEOUT = 7200; // 10 minutes + + private $title = ''; + private $threadurl = ''; + private $version; // Holds the XenForo version + + public function getName() { + + switch($this->queriedContext) { + case self::CONTEXT_THREAD: return $this->title . ' - ' . static::NAME; + } + + return parent::getName(); + + } + + public function getURI() { + + switch($this->queriedContext) { + case self::CONTEXT_THREAD: return $this->threadurl; + } + + return parent::getURI(); + + } + + public function collectData() { + + $this->threadurl = filter_var( + $this->getInput('url'), + FILTER_VALIDATE_URL, + FILTER_FLAG_SCHEME_REQUIRED | FILTER_FLAG_HOST_REQUIRED | FILTER_FLAG_PATH_REQUIRED); + + if($this->threadurl === false) { + returnClientError('The URL you provided is invalid!'); + } + + $urlparts = parse_url($this->threadurl, PHP_URL_SCHEME); + + // Scheme must be "http" or "https" + if(preg_match('/http[s]{0,1}/', parse_url($this->threadurl, PHP_URL_SCHEME)) == false) { + returnClientError('The URL you provided doesn\'t specify a valid scheme (http or https)!'); + } + + // Path cannot be root (../) + if(parse_url($this->threadurl, PHP_URL_PATH) === '/') { + returnClientError('The URL you provided doesn\'t link to a valid thread (root path)!'); + } + + // XenForo adds a thread ID to the URL, like "...-thread.454934283". It must be present + if(preg_match('/.+\.\d+[\/]{0,1}/', parse_URL($this->threadurl, PHP_URL_PATH)) == false) { + returnClientError('The URL you provided doesn\'t link to a valid thread (ID missing)!'); + } + + // We want to start at the first page in the thread. XenForo uses "../page-n" syntax + // to identify pages (except for the first page). + // Notice: XenForo uses the concept of "sentinels" to find and replace parts in the + // URL. Technically forum hosts can change the syntax! + if(preg_match('/.+\/(page-\d+.*)$/', $this->threadurl, $matches) != false) { + + // before: https://xenforo.com/community/threads/guide-to-suggestions.2285/page-5 + // after : https://xenforo.com/community/threads/guide-to-suggestions.2285/ + $this->threadurl = str_replace($matches[1], '', $this->threadurl); + + } + + $html = getSimpleHTMLDOMCached($this->threadurl) + or returnServerError('Failed loading data from "' . $this->threadurl . '"!'); + + $html = defaultLinkTo($html, $this->threadurl); + + // Notice: The DOM structure changes depending on the XenForo version used + if($mainContent = $html->find('div.mainContent', 0)) { + $this->version = self::XENFORO_VERSION_1; + } elseif ($mainContent = $html->find('div[class="p-body"]', 0)) { + $this->version = self::XENFORO_VERSION_2; + } else { + returnServerError('This forum is currently not supported!'); + } + + switch($this->version) { + case self::XENFORO_VERSION_1: + + $titleBar = $mainContent->find('div.titleBar h1', 0) + or returnServerError('Error finding title bar!'); + + $this->title = $titleBar->plaintext; + + // Store items from current page (we'll use $this->items as LIFO buffer) + $this->extractThreadPostsV1($html, $this->threadurl); + $this->extractPagesV1($html); + + break; + + case self::XENFORO_VERSION_2: + + $titleBar = $mainContent->find('div[class="p-title"] h1', 0) + or returnServerError('Error finding title bar!'); + + $this->title = $titleBar->plaintext; + $this->extractThreadPostsV2($html, $this->threadurl); + $this->extractPagesV2($html); + + break; + } + + while(count($this->items) > $this->getInput('limit')) { + array_shift($this->items); + } + + } + + /** + * Extracts thread posts + * @param $html A simplehtmldom object + * @param $url The url from which $html was loaded + */ + private function extractThreadPostsV1($html, $url) { + + $lang = $html->find('html', 0)->lang; + + // Posts are contained in an "ol" + $messageList = $html->find('#messageList li') + or returnServerError('Error finding message list!'); + + foreach($messageList as $post) { + + if(!isset($post->attr['id'])) { // Skip ads + continue; + } + + $item = array(); + + $item['uri'] = $url . '#' . $post->getAttribute('id'); + + $content = $post->find('.messageContent article', 0); + + // Add some style to quotes + foreach($content->find('.bbCodeQuote') as $quote) { + $quote->style = ' + color: #495566; + background-color: rgb(248,251,253); + border: 1px solid rgb(111, 140, 180); + border-color: rgb(111, 140, 180); + font-style: italic;'; + } + + // Remove script tags + foreach($content->find('script') as $script) { + $script->outertext = ''; + } + + $item['content'] = $content->innertext; + + // Remove quotes (for the title) + foreach($content->find('.bbCodeQuote') as $quote) { + $quote->innertext = ''; + } + + $title = trim($content->plaintext); + + if(strlen($title) > 70) { + $item['title'] = substr($title, 0, strpos($title, ' ', 70)) . '...'; + } else { + $item['title'] = $title; + } + + /** + * Timestamps are presented in two forms: + * + * 1) short version (for older posts?) + * <span + * class="DateTime" + * title="22 Oct. 2018 at 23:47" + * >22 Oct. 2018</span> + * + * This form has to be interpreted depending on the current language. + * + * 2) long version (for newer posts?) + * <abbr + * class="DateTime" + * data-time="1541008785" + * data-diff="310694" + * data-datestring="31 Oct. 2018" + * data-timestring="18:59" + * title="31 Oct. 2018 at 18:59" + * >Wednesday at 18:59</abbr> + * + * This form has the timestamp embedded (data-time) + */ + if($timestamp = $post->find('abbr.DateTime', 0)) { // long version (preffered) + $item['timestamp'] = $timestamp->{'data-time'}; + } elseif($timestamp = $post->find('span.DateTime', 0)) { // short version + $item['timestamp'] = $this->fixDate($timestamp->title, $lang); + } + + $item['author'] = $post->getAttribute('data-author'); + + // Bridge specific properties + $item['id'] = $post->getAttribute('id'); + + $this->items[] = $item; + + } + + } + + private function extractThreadPostsV2($html, $url) { + + $lang = $html->find('html', 0)->lang; + + $messageList = $html->find('div[class="block-body"] article') + or returnServerError('Error finding message list!'); + + foreach($messageList as $post) { + + if(!isset($post->attr['id'])) { // Skip ads + continue; + } + + $item = array(); + + $item['uri'] = $url . '#' . $post->getAttribute('id'); + + $title = $post->find('div[class="message-content"] article', 0)->plaintext; + $end = strpos($title, ' ', 70); + $item['title'] = substr($title, 0, $end); + + $item['timestamp'] = $this->fixDate($post->find('time', 0)->title, $lang); + $item['author'] = $post->getAttribute('data-author'); + $item['content'] = $post->find('div[class="message-content"] article', 0); + + // Bridge specific properties + $item['id'] = $post->getAttribute('id'); + + $this->items[] = $item; + + } + + } + + private function extractPagesV1($html) { + + // A navigation bar becomes available if the number of posts grows too + // high. When this happens we need to load further pages (from last backwards) + if(($pageNav = $html->find('div.PageNav', 0))) { + + $lastpage = $pageNav->{'data-last'}; + $baseurl = $pageNav->{'data-baseurl'}; + $sentinel = $pageNav->{'data-sentinel'}; + + $hosturl = parse_url($this->threadurl, PHP_URL_SCHEME) + . '://' + . parse_url($this->threadurl, PHP_URL_HOST) + . '/'; + + $page = $lastpage; + + // Load at least the last page + do { + + $pageurl = $hosturl . str_replace($sentinel, $lastpage, $baseurl); + + // We can optimize performance by caching all but the last page + if($page != $lastpage) { + $html = getSimpleHTMLDOMCached($pageurl) + or returnServerError('Error loading contents from ' . $pageurl . '!'); + } else { + $html = getSimpleHTMLDOM($pageurl) + or returnServerError('Error loading contents from ' . $pageurl . '!'); + } + + $html = defaultLinkTo($html, $hosturl); + + $this->extractThreadPostsV1($html, $pageurl); + + $page--; + + } while (count($this->items) < $this->getInput('limit') && $page != 1); + + } + + } + + private function extractPagesV2($html) { + + // A navigation bar becomes available if the number of posts grows too + // high. When this happens we need to load further pages (from last backwards) + if(($pageNav = $html->find('div.pageNav', 0))) { + + foreach($pageNav->find('li') as $nav) { + $lastpage = $nav->plaintext; + } + + // Manually extract baseurl and inject sentinel + $baseurl = $pageNav->find('li a', -1)->href; + $baseurl = str_replace('page-' . $lastpage, 'page-{{sentinel}}', $baseurl); + + $sentinel = '{{sentinel}}'; + + $hosturl = parse_url($this->threadurl, PHP_URL_SCHEME) + . '://' + . parse_url($this->threadurl, PHP_URL_HOST); + + $page = $lastpage; + + // Load at least the last page + do { + + $pageurl = $hosturl . str_replace($sentinel, $lastpage, $baseurl); + + // We can optimize performance by caching all but the last page + if($page != $lastpage) { + $html = getSimpleHTMLDOMCached($pageurl) + or returnServerError('Error loading contents from ' . $pageurl . '!'); + } else { + $html = getSimpleHTMLDOM($pageurl) + or returnServerError('Error loading contents from ' . $pageurl . '!'); + } + + $html = defaultLinkTo($html, $this->hosturl); + + $this->extractThreadPostsV2($html, $this->pageurl); + + $page--; + + } while (count($this->items) < $this->getInput('limit') && $page != 1); + + } + + } + + /** + * Fixes dates depending on the choosen language: + * + * de : dd.mm.yy + * en : dd.mm.yy + * it : dd/mm/yy + * + * Basically strtotime doesn't convert dates correctly due to formats + * being hard to interpret. So we use the DateTime object. + * + * We don't know the timezone, so just assume +00:00 (or whatever + * DateTime chooses) + */ + private function fixDate($date, $lang = 'en-US') { + + $mnamesen = [ + 'January', + 'Feburary', + 'March', + 'April', + 'May', + 'June', + 'July', + 'August', + 'September', + 'October', + 'November', + 'December' + ]; + + switch($lang) { + case 'en-US': // example: Jun 9, 2018 at 11:46 PM + + $df = date_create_from_format('M d, Y \a\t H:i A', $date); + break; + + case 'de-DE': // example: 19 Juli 2018 um 19:27 Uhr + + $mnamesde = [ + 'Januar', + 'Februar', + 'März', + 'April', + 'Mai', + 'Juni', + 'Juli', + 'August', + 'September', + 'Oktober', + 'November', + 'Dezember' + ]; + + $mnamesdeshort = [ + 'Jan.', + 'Feb.', + 'Mär.', + 'Apr.', + 'Mai', + 'Juni', + 'Juli', + 'Aug.', + 'Sep.', + 'Okt.', + 'Nov.', + 'Dez.' + ]; + + $date = str_ireplace($mnamesde, $mnamesen, $date); + $date = str_ireplace($mnamesdeshort, $mnamesen, $date); + + $df = date_create_from_format('d M Y \u\m H:i \U\h\r', $date); + break; + + } + + // debugMessage(date_format($df, 'U')); + + return date_format($df, 'U'); + + } + +} diff --git a/bridges/YGGTorrentBridge.php b/bridges/YGGTorrentBridge.php index bc434d3..f057d87 100644 --- a/bridges/YGGTorrentBridge.php +++ b/bridges/YGGTorrentBridge.php @@ -101,7 +101,7 @@ class YGGTorrentBridge extends BridgeAbstract { . $category . '&sub_category=' . $subcategory - . '&do=search') + . '&do=search&order=desc&sort=publish_date') or returnServerError('Unable to query Yggtorrent !'); $count = 0; @@ -110,8 +110,8 @@ class YGGTorrentBridge extends BridgeAbstract { foreach($results->find('tr') as $row) { $count++; - if($count == 1) continue; - if($count == 12) break; + if($count == 1) continue; // Skip table header + if($count == 22) break; // Stop processing after 21 items (20 + 1 table header) $item = array(); $item['timestamp'] = $row->find('.hidden', 1)->plaintext; $item['title'] = $row->find('a', 1)->plaintext; @@ -127,7 +127,7 @@ class YGGTorrentBridge extends BridgeAbstract { } - public function collectTorrentData($url) { + private function collectTorrentData($url) { //For weird reason, the link we get can be invalid, we fix it. $url_full = explode('/', $url); @@ -135,7 +135,7 @@ class YGGTorrentBridge extends BridgeAbstract { $url_full[5] = urlencode($url_full[5]); $url_full[6] = urlencode($url_full[6]); $url = implode('/', $url_full); - $page = getSimpleHTMLDOM($url) or returnServerError('Unable to query Yggtorrent page !'); + $page = getSimpleHTMLDOMCached($url) or returnServerError('Unable to query Yggtorrent page !'); $author = $page->find('.informations', 0)->find('a', 4)->plaintext; $content = $page->find('.default', 1); return array('author' => $author, 'content' => $content); diff --git a/bridges/YoutubeBridge.php b/bridges/YoutubeBridge.php index e597fe3..67e9566 100644 --- a/bridges/YoutubeBridge.php +++ b/bridges/YoutubeBridge.php @@ -45,9 +45,25 @@ class YoutubeBridge extends BridgeAbstract { 'type' => 'number', 'exampleValue' => 1 ) + ), + 'global' => array( + 'duration_min' => array( + 'name' => 'min. duration (minutes)', + 'type' => 'number', + 'title' => 'Minimum duration for the video in minutes', + 'exampleValue' => 5 + ), + 'duration_max' => array( + 'name' => 'max. duration (minutes)', + 'type' => 'number', + 'title' => 'Maximum duration for the video in minutes', + 'exampleValue' => 10 + ) ) ); + private $feedName = ''; + private function ytBridgeQueryVideoInfo($vid, &$author, &$desc, &$time){ $html = $this->ytGetSimpleHTMLDOM(self::URI . "watch?v=$vid"); @@ -113,6 +129,17 @@ class YoutubeBridge extends BridgeAbstract { private function ytBridgeParseHtmlListing($html, $element_selector, $title_selector, $add_parsed_items = true) { $limit = $add_parsed_items ? 10 : INF; $count = 0; + + $duration_min = $this->getInput('duration_min') ?: -1; + $duration_min = $duration_min * 60; + + $duration_max = $this->getInput('duration_max') ?: INF; + $duration_max = $duration_max * 60; + + if($duration_max < $duration_min) { + returnClientError('Max duration must be greater than min duration!'); + } + foreach($html->find($element_selector) as $element) { if($count < $limit) { $author = ''; @@ -120,14 +147,33 @@ class YoutubeBridge extends BridgeAbstract { $time = 0; $vid = str_replace('/watch?v=', '', $element->find('a', 0)->href); $vid = substr($vid, 0, strpos($vid, '&') ?: strlen($vid)); - $title = $this->ytBridgeFixTitle($element->find($title_selector, 0)->plaintext); - if($title != '[Private Video]' && strpos($vid, 'googleads') === false) { - if ($add_parsed_items) { - $this->ytBridgeQueryVideoInfo($vid, $author, $desc, $time); - $this->ytBridgeAddItem($vid, $title, $author, $desc, $time); - } - $count++; + $title = trim($this->ytBridgeFixTitle($element->find($title_selector, 0)->plaintext)); + + if (strpos($vid, 'googleads') !== false + || $title == '[Private video]' + || $title == '[Deleted video]' + ) { + continue; + } + + // The duration comes in one of the formats: + // hh:mm:ss / mm:ss / m:ss + // 01:03:30 / 15:06 / 1:24 + $durationText = trim($element->find('div.timestamp span', 0)->plaintext); + $durationText = preg_replace('/([\d]{1,2})\:([\d]{2})/', '00:$1:$2', $durationText); + + sscanf($durationText, '%d:%d:%d', $hours, $minutes, $seconds); + $duration = $hours * 3600 + $minutes * 60 + $seconds; + + if($duration < $duration_min || $duration > $duration_max) { + continue; } + + if ($add_parsed_items) { + $this->ytBridgeQueryVideoInfo($vid, $author, $desc, $time); + $this->ytBridgeAddItem($vid, $title, $author, $desc, $time); + } + $count++; } } return $count; @@ -140,7 +186,9 @@ class YoutubeBridge extends BridgeAbstract { private function ytGetSimpleHTMLDOM($url){ return getSimpleHTMLDOM($url, - $header = array(), + $header = array( + 'Accept-Language: en-US' + ), $opts = array(), $lowercase = true, $forceTagsClosed = true, @@ -168,7 +216,7 @@ class YoutubeBridge extends BridgeAbstract { } if(!empty($url_feed) && !empty($url_listing)) { - if($xml = $this->ytGetSimpleHTMLDOM($url_feed)) { + if(!$this->skipFeeds() && $xml = $this->ytGetSimpleHTMLDOM($url_feed)) { $this->ytBridgeParseXmlFeed($xml); } elseif($html = $this->ytGetSimpleHTMLDOM($url_listing)) { $this->ytBridgeParseHtmlListing($html, 'li.channels-content-item', 'h3'); @@ -182,7 +230,7 @@ class YoutubeBridge extends BridgeAbstract { $html = $this->ytGetSimpleHTMLDOM($url_listing) or returnServerError("Could not request YouTube. Tried:\n - $url_listing"); $item_count = $this->ytBridgeParseHtmlListing($html, 'tr.pl-video', '.pl-video-title a', false); - if ($item_count <= 15 && ($xml = $this->ytGetSimpleHTMLDOM($url_feed))) { + if ($item_count <= 15 && !$this->skipFeeds() && ($xml = $this->ytGetSimpleHTMLDOM($url_feed))) { $this->ytBridgeParseXmlFeed($xml); } else { $this->ytBridgeParseHtmlListing($html, 'tr.pl-video', '.pl-video-title a'); @@ -215,6 +263,10 @@ class YoutubeBridge extends BridgeAbstract { } } + private function skipFeeds() { + return ($this->getInput('duration_min') || $this->getInput('duration_max')); + } + public function getName(){ // Name depends on queriedContext: switch($this->queriedContext) { diff --git a/bridges/ZDNetBridge.php b/bridges/ZDNetBridge.php index 86e4b49..75df3b1 100644 --- a/bridges/ZDNetBridge.php +++ b/bridges/ZDNetBridge.php @@ -1,9 +1,9 @@ <?php -class ZDNetBridge extends BridgeAbstract { +class ZDNetBridge extends FeedExpander { const MAINTAINER = 'ORelio'; const NAME = 'ZDNet Bridge'; - const URI = 'http://www.zdnet.com/'; + const URI = 'https://www.zdnet.com/'; const DESCRIPTION = 'Technology News, Analysis, Comments and Product Reviews for IT Professionals.'; //http://www.zdnet.com/zdnet.opml @@ -160,143 +160,42 @@ class ZDNetBridge extends BridgeAbstract { )); public function collectData(){ - - function stripCdata($string){ - $string = str_replace('<![CDATA[', '', $string); - $string = str_replace(']]>', '', $string); - return trim($string); - } - - function extractFromDelimiters($string, $start, $end){ - if(strpos($string, $start) !== false) { - $section_retrieved = substr($string, strpos($string, $start) + strlen($start)); - $section_retrieved = substr($section_retrieved, 0, strpos($section_retrieved, $end)); - return $section_retrieved; - } - - return false; - } - - function stripWithDelimiters($string, $start, $end){ - while(strpos($string, $start) !== false) { - $section_to_remove = substr($string, strpos($string, $start)); - $section_to_remove = substr($section_to_remove, 0, strpos($section_to_remove, $end) + strlen($end)); - $string = str_replace($section_to_remove, '', $string); - } - - return $string; - } - - function stripRecursiveHtmlSection($string, $tag_name, $tag_start){ - $open_tag = '<' . $tag_name; - $close_tag = '</' . $tag_name . '>'; - $close_tag_length = strlen($close_tag); - if(strpos($tag_start, $open_tag) === 0) { - while(strpos($string, $tag_start) !== false) { - $max_recursion = 100; - $section_to_remove = null; - $section_start = strpos($string, $tag_start); - $search_offset = $section_start; - do { - $max_recursion--; - $section_end = strpos($string, $close_tag, $search_offset); - $search_offset = $section_end + $close_tag_length; - $section_to_remove = substr( - $string, - $section_start, - $section_end - $section_start + $close_tag_length - ); - - $open_tag_count = substr_count($section_to_remove, $open_tag); - $close_tag_count = substr_count($section_to_remove, $close_tag); - } while ($open_tag_count > $close_tag_count && $max_recursion > 0); - $string = str_replace($section_to_remove, '', $string); - } - } - return $string; - } - - $baseUri = self::URI; + $baseUri = static::URI; $feed = $this->getInput('feed'); if(strpos($feed, 'downloads!') !== false) { $feed = str_replace('downloads!', '', $feed); $baseUri = str_replace('www.', 'downloads.', $baseUri); } $url = $baseUri . trim($feed, '/') . '/rss.xml'; - $html = getSimpleHTMLDOM($url) - or returnServerError('Could not request ZDNet: ' . $url); - $limit = 0; - - foreach($html->find('item') as $element) { - if($limit < 10) { - $article_url = preg_replace( - '/([^#]+)#ftag=.*/', - '$1', - stripCdata(extractFromDelimiters($element->innertext, '<link>', '</link>')) - ); - - $article_author = stripCdata(extractFromDelimiters($element->innertext, 'role="author">', '<')); - $article_title = stripCdata($element->find('title', 0)->plaintext); - $article_subtitle = stripCdata($element->find('description', 0)->plaintext); - $article_timestamp = strtotime(stripCdata($element->find('pubDate', 0)->plaintext)); - $article = getSimpleHTMLDOM($article_url) - or returnServerError('Could not request ZDNet: ' . $article_url); - - if(!empty($article_author)) { - $author = $article_author; - } else { - $author = $article->find('meta[name=author]', 0); - if(is_object($author)) { - $author = $author->content; - } else { - $author = 'ZDNet'; - } - } - - $thumbnail = $article->find('meta[itemprop=image]', 0); - if(is_object($thumbnail)) { - $thumbnail = $thumbnail->content; - } else { - $thumbnail = ''; - } - - $contents = $article->find('article', 0)->innertext; - foreach(array( - '<div class="shareBar"', - '<div class="shortcodeGalleryWrapper"', - '<div class="relatedContent', - '<div class="downloadNow', - '<div data-shortcode', - '<div id="sharethrough', - '<div id="inpage-video' - ) as $div_start) { - $contents = stripRecursiveHtmlSection($contents, 'div', $div_start); - } - $contents = stripWithDelimiters($contents, '<script', '</script>'); - $contents = stripWithDelimiters($contents, '<meta itemprop="image"', '>'); - $contents = trim(stripWithDelimiters($contents, '<section class="sharethrough-top', '</section>')); - $content_img = strpos($contents, '<img'); //Look for first image - if (($content_img !== false && $content_img < 512) || $thumbnail == '') { - $content_img = ''; //Image already present on article beginning or no thumbnail - } else { - $content_img = '<p><img src="'.$thumbnail.'" /></p>'; //Include thumbnail - } - $contents = $content_img - . '<p><b>' - . $article_subtitle - . '</b></p>' - . $contents; + $this->collectExpandableDatas($url); + } - $item = array(); - $item['author'] = $author; - $item['uri'] = $article_url; - $item['title'] = $article_title; - $item['timestamp'] = $article_timestamp; - $item['content'] = $contents; - $this->items[] = $item; - $limit++; - } + protected function parseItem($item){ + $item = parent::parseItem($item); + + $article = getSimpleHTMLDOMCached($item['uri']); + if(!$article) + returnServerError('Could not request ZDNet: ' . $url); + + $contents = $article->find('article', 0)->innertext; + foreach(array( + '<div class="shareBar"', + '<div class="shortcodeGalleryWrapper"', + '<div class="relatedContent', + '<div class="downloadNow', + '<div data-shortcode', + '<div id="sharethrough', + '<div id="inpage-video' + ) as $div_start) { + $contents = stripRecursiveHtmlSection($contents, 'div', $div_start); } + $contents = stripWithDelimiters($contents, '<script', '</script>'); + $contents = stripWithDelimiters($contents, '<meta itemprop="image"', '>'); + $contents = stripWithDelimiters($contents, '<svg class="svg-symbol', '</svg>'); + $contents = trim(stripWithDelimiters($contents, '<section class="sharethrough-top', '</section>')); + $item['content'] = $contents; + + return $item; } } diff --git a/bridges/ZoneTelechargementBridge.php b/bridges/ZoneTelechargementBridge.php new file mode 100644 index 0000000..670a50c --- /dev/null +++ b/bridges/ZoneTelechargementBridge.php @@ -0,0 +1,88 @@ +<?php +class ZoneTelechargementBridge extends BridgeAbstract { + const NAME = 'Zone Telechargement'; + const URI = 'https://www.zone-telechargement1.org/'; + const DESCRIPTION = 'Suivi de série sur Zone Telechargement'; + const MAINTAINER = 'sysadminstory'; + const PARAMETERS = array( + 'Suivre la publication des épisodes d\'une série en cours de diffusion' => array( + 'url' => array( + 'name' => 'URL de la série', + 'type' => 'text', + 'required' => true, + 'title' => 'URL d\'une série sans le https://ww4.zone-telechargement1.org/', + 'exampleValue' => 'telecharger-series/31079-halt-and-catch-fire-saison-4-french-hd720p.html' + ) + ) + ); + + public function getIcon() { + return 'https://ww7.zone-telechargement1.org/templates/Default/images/favicon.ico'; + } + + public function collectData(){ + $html = getSimpleHTMLDOM(self::URI . $this->getInput('url')) + or returnServerError('Could not request Zone Telechargement.'); + + // Get the TV show title + $qualityselector = 'div[style=font-size: 18px;margin: 10px auto;color:red;font-weight:bold;text-align:center;]'; + $show = trim($html->find('div[class=smallsep]', 0)->next_sibling()->plaintext); + $quality = trim(explode("\n", $html->find($qualityselector, 0)->plaintext)[0]); + $this->showTitle = $show . ' ' . $quality; + + // Get the post content + $linkshtml = $html->find('div[class=postinfo]', 0); + + $episodes = array(); + + $list = $linkshtml->find('a'); + // Construct the tabble of episodes using the links + foreach($list as $element) { + // Retrieve episode number from link text + $epnumber = explode(' ', $element->plaintext)[1]; + $hoster = $this->findLinkHoster($element); + + // Format the link and add the link to the corresponding episode table + $episodes[$epnumber][] = '<a href="' . $element->href . '">' . $hoster . ' - ' + . $this->showTitle . ' Episode ' . $epnumber . '</a>'; + + } + + // Finally construct the items array + foreach($episodes as $epnum => $episode) { + $item = array(); + // Add every link available in the episode table separated by a <br/> tag + $item['content'] = implode('<br/>', $episode); + $item['title'] = $this->showTitle . ' Episode ' . $epnum; + // As RSS Bridge use the URI as GUID they need to be unique : adding a md5 hash of the title element + // should geneerate unique URI to prevent confusion for RSS readers + $item['uri'] = self::URI . $this->getInput('url') . '#' . hash('md5', $item['title']); + // Insert the episode at the beginning of the item list, to show the newest episode first + array_unshift($this->items, $item); + } + } + + public function getName() { + switch($this->queriedContext) { + case 'Suivre la publication des épisodes d\'une série en cours de diffusion': + return $this->showTitle . ' - ' . self::NAME; + break; + default: + return self::NAME; + } + } + + private function findLinkHoster($element) { + // The hoster name is one level higher than the link tag : get the parent element + $element = $element->parent(); + //echo "PARENT : $element \n"; + $continue = true; + // Walk through all elements in the reverse order until finding the one with a div and that is not a <br/> + while(!($element->find('div', 0) != null && $element->tag != 'br')) { + $element = $element->prev_sibling(); + } + // Return the text of the div : it's the file hoster name ! + return $element->find('div', 0)->plaintext; + + } +} diff --git a/caches/FileCache.php b/caches/FileCache.php index de17d52..04d08a2 100644 --- a/caches/FileCache.php +++ b/caches/FileCache.php @@ -27,6 +27,7 @@ class FileCache implements CacheInterface { public function getTime(){ $cacheFile = $this->getCacheFile(); + clearstatcache(false, $cacheFile); if(file_exists($cacheFile)) { return filemtime($cacheFile); } diff --git a/config.default.ini.php b/config.default.ini.php index 5909ad8..2d6fca1 100644 --- a/config.default.ini.php +++ b/config.default.ini.php @@ -11,6 +11,12 @@ ; false = disabled (default) custom_timeout = false +[admin] +; Advertise an email address where people can reach the administrator. +; This address is displayed on the main page, visible to everyone! +; "" = Disabled (default) +email = "" + [proxy] ; Sets the proxy url (i.e. "tcp://192.168.0.0:32") diff --git a/formats/AtomFormat.php b/formats/AtomFormat.php index 9bd08bd..044e433 100644 --- a/formats/AtomFormat.php +++ b/formats/AtomFormat.php @@ -11,14 +11,18 @@ class AtomFormat extends FormatAbstract{ $httpHost = isset($_SERVER['HTTP_HOST']) ? $_SERVER['HTTP_HOST'] : ''; $httpInfo = isset($_SERVER['PATH_INFO']) ? $_SERVER['PATH_INFO'] : ''; - $serverRequestUri = $this->xml_encode($_SERVER['REQUEST_URI']); + $serverRequestUri = isset($_SERVER['REQUEST_URI']) ? $this->xml_encode($_SERVER['REQUEST_URI']) : ''; $extraInfos = $this->getExtraInfos(); $title = $this->xml_encode($extraInfos['name']); - $uri = !empty($extraInfos['uri']) ? $extraInfos['uri'] : 'https://github.com/RSS-Bridge/rss-bridge'; + $uri = !empty($extraInfos['uri']) ? $extraInfos['uri'] : REPOSITORY; $uriparts = parse_url($uri); - $icon = $this->xml_encode($uriparts['scheme'] . '://' . $uriparts['host'] .'/favicon.ico'); + if(!empty($extraInfos['icon'])) { + $icon = $extraInfos['icon']; + } else { + $icon = $this->xml_encode($uriparts['scheme'] . '://' . $uriparts['host'] . '/favicon.ico'); + } $uri = $this->xml_encode($uri); @@ -35,7 +39,7 @@ class AtomFormat extends FormatAbstract{ foreach($item['enclosures'] as $enclosure) { $entryEnclosures .= '<link rel="enclosure" href="' . $this->xml_encode($enclosure) - . '"/>' + . '" type="' . getMimeType($enclosure) . '" />' . PHP_EOL; } } diff --git a/formats/HtmlFormat.php b/formats/HtmlFormat.php index c4c5012..71f70d2 100644 --- a/formats/HtmlFormat.php +++ b/formats/HtmlFormat.php @@ -48,7 +48,7 @@ class HtmlFormat extends FormatAbstract { } $entryCategories = ''; - if(isset($item['categories'])) { + if(isset($item['categories']) && count($item['categories']) > 0) { $entryCategories = '<div class="categories"><p>Categories:</p>'; foreach($item['categories'] as $category) { @@ -85,6 +85,8 @@ EOD; <meta charset="{$charset}"> <title>{$title}</title> <link href="static/HtmlFormat.css" rel="stylesheet"> + <link rel="alternate" type="application/atom+xml" title="Atom" href="./?{$atomquery}" /> + <link rel="alternate" type="application/rss+xml" title="RSS" href="/?{$mrssquery}" /> <meta name="robots" content="noindex, follow"> </head> <body> diff --git a/formats/MrssFormat.php b/formats/MrssFormat.php index 6d07928..9110e02 100644 --- a/formats/MrssFormat.php +++ b/formats/MrssFormat.php @@ -10,7 +10,7 @@ class MrssFormat extends FormatAbstract { $httpHost = isset($_SERVER['HTTP_HOST']) ? $_SERVER['HTTP_HOST'] : ''; $httpInfo = isset($_SERVER['PATH_INFO']) ? $_SERVER['PATH_INFO'] : ''; - $serverRequestUri = $this->xml_encode($_SERVER['REQUEST_URI']); + $serverRequestUri = isset($_SERVER['REQUEST_URI']) ? $this->xml_encode($_SERVER['REQUEST_URI']) : ''; $extraInfos = $this->getExtraInfos(); $title = $this->xml_encode($extraInfos['name']); @@ -18,11 +18,11 @@ class MrssFormat extends FormatAbstract { if(!empty($extraInfos['uri'])) { $uri = $this->xml_encode($extraInfos['uri']); } else { - $uri = 'https://github.com/RSS-Bridge/rss-bridge'; + $uri = REPOSITORY; } $uriparts = parse_url($uri); - $icon = $this->xml_encode($uriparts['scheme'] . '://' . $uriparts['host'] .'/favicon.ico'); + $icon = $this->xml_encode($uriparts['scheme'] . '://' . $uriparts['host'] . '/favicon.ico'); $items = ''; foreach($this->getItems() as $item) { @@ -37,7 +37,7 @@ class MrssFormat extends FormatAbstract { if(isset($item['enclosures'])) { $entryEnclosures .= '<enclosure url="' . $this->xml_encode($item['enclosures'][0]) - . '"/>'; + . '" type="' . getMimeType($item['enclosures'][0]) . '" />'; if(count($item['enclosures']) > 1) { $entryEnclosures .= PHP_EOL; @@ -45,7 +45,7 @@ class MrssFormat extends FormatAbstract { Some media files might not be shown to you. Consider using the ATOM format instead!'; foreach($item['enclosures'] as $enclosure) { $entryEnclosures .= '<atom:link rel="enclosure" href="' - . $enclosure . '" />' + . $enclosure . '" type="' . getMimeType($enclosure) . '" />' . PHP_EOL; } } @@ -56,7 +56,7 @@ Some media files might not be shown to you. Consider using the ATOM format inste foreach($item['categories'] as $category) { $entryCategories .= '<category>' - . $category . '</category>' + . $category . '</category>' . PHP_EOL; } } @@ -79,6 +79,8 @@ EOD; $charset = $this->getCharset(); + /* xml attributes need to have certain characters escaped to be w3c compliant */ + $imageTitle = htmlspecialchars($title, ENT_COMPAT); /* Data are prepared, now let's begin the "MAGIE !!!" */ $toReturn = <<<EOD <?xml version="1.0" encoding="{$charset}"?> @@ -90,7 +92,7 @@ xmlns:atom="http://www.w3.org/2005/Atom"> <title>{$title}</title> <link>http{$https}://{$httpHost}{$httpInfo}/</link> <description>{$title}</description> - <image url="{$icon}" title="{$title}" link="{$uri}"/> + <image url="{$icon}" title="{$imageTitle}" link="{$uri}"/> <atom:link rel="alternate" type="text/html" href="{$uri}" /> <atom:link rel="self" href="http{$https}://{$httpHost}{$serverRequestUri}" /> {$items} @@ -1,29 +1,4 @@ <?php -require_once __DIR__ . '/lib/RssBridge.php'; - -define('PHP_VERSION_REQUIRED', '5.6.0'); - -// Specify directory for cached files (using FileCache) -define('CACHE_DIR', __DIR__ . '/cache'); - -// Specify path for whitelist file -define('WHITELIST_FILE', __DIR__ . '/whitelist.txt'); - -Configuration::verifyInstallation(); -Configuration::loadConfiguration(); - -Authentication::showPromptIfNeeded(); - -date_default_timezone_set('UTC'); -error_reporting(0); - -/* -Move the CLI arguments to the $_GET array, in order to be able to use -rss-bridge from the command line -*/ -parse_str(implode('&', array_slice($argv, 1)), $cliArgs); -$params = array_merge($_GET, $cliArgs); - /* Create a file named 'DEBUG' for enabling debug mode. For further security, you may put whitelisted IP addresses in the file, @@ -36,22 +11,53 @@ if(file_exists('DEBUG')) { $debug_whitelist = trim(file_get_contents('DEBUG')); $debug_enabled = empty($debug_whitelist) - || in_array($_SERVER['REMOTE_ADDR'], explode("\n", $debug_whitelist)); + || in_array($_SERVER['REMOTE_ADDR'], + explode("\n", str_replace("\r", '', $debug_whitelist) + ) + ); if($debug_enabled) { ini_set('display_errors', '1'); error_reporting(E_ALL); define('DEBUG', true); + if (empty($debug_whitelist)) { + define('DEBUG_INSECURE', true); + } } } -// FIXME : beta test UA spoofing, please report any blacklisting by PHP-fopen-unfriendly websites +require_once __DIR__ . '/lib/RssBridge.php'; -$userAgent = 'Mozilla/5.0(X11; Linux x86_64; rv:30.0)'; -$userAgent .= ' Gecko/20121202 Firefox/30.0(rss-bridge/0.1;'; -$userAgent .= '+https://github.com/RSS-Bridge/rss-bridge)'; +// Specify path for whitelist file +define('WHITELIST_FILE', __DIR__ . '/whitelist.txt'); -ini_set('user_agent', $userAgent); +Configuration::verifyInstallation(); +Configuration::loadConfiguration(); + +Authentication::showPromptIfNeeded(); + +date_default_timezone_set('UTC'); + +/* +Move the CLI arguments to the $_GET array, in order to be able to use +rss-bridge from the command line +*/ +if (isset($argv)) { + parse_str(implode('&', array_slice($argv, 1)), $cliArgs); + $params = array_merge($_GET, $cliArgs); +} else { + $params = $_GET; +} + +define('USER_AGENT', + 'Mozilla/5.0 (X11; Linux x86_64; rv:30.0) Gecko/20121202 Firefox/30.0(rss-bridge/' + . Configuration::$VERSION + . ';+' + . REPOSITORY + . ')' +); + +ini_set('user_agent', USER_AGENT); // default whitelist $whitelist_default = array( @@ -74,9 +80,9 @@ $whitelist_default = array( try { - Bridge::setDir(__DIR__ . '/bridges/'); - Format::setDir(__DIR__ . '/formats/'); - Cache::setDir(__DIR__ . '/caches/'); + Bridge::setDir(PATH_LIB_BRIDGES); + Format::setDir(PATH_LIB_FORMATS); + Cache::setDir(PATH_LIB_CACHES); if(!file_exists(WHITELIST_FILE)) { $whitelist_selection = $whitelist_default; @@ -95,10 +101,51 @@ try { $whitelist_selection = array_map('strtolower', $whitelist_selection); } + $showInactive = filter_input(INPUT_GET, 'show_inactive', FILTER_VALIDATE_BOOLEAN); $action = array_key_exists('action', $params) ? $params['action'] : null; $bridge = array_key_exists('bridge', $params) ? $params['bridge'] : null; - if($action === 'display' && !empty($bridge)) { + // Return list of bridges as JSON formatted text + if($action === 'list') { + + $list = new StdClass(); + $list->bridges = array(); + $list->total = 0; + + foreach(Bridge::listBridges() as $bridgeName) { + + $bridge = Bridge::create($bridgeName); + + if($bridge === false) { // Broken bridge, show as inactive + + $list->bridges[$bridgeName] = array( + 'status' => 'inactive' + ); + + continue; + + } + + $status = Bridge::isWhitelisted($whitelist_selection, strtolower($bridgeName)) ? 'active' : 'inactive'; + + $list->bridges[$bridgeName] = array( + 'status' => $status, + 'uri' => $bridge->getURI(), + 'name' => $bridge->getName(), + 'icon' => $bridge->getIcon(), + 'parameters' => $bridge->getParameters(), + 'maintainer' => $bridge->getMaintainer(), + 'description' => $bridge->getDescription() + ); + + } + + $list->total = count($list->bridges); + + header('Content-Type: application/json'); + echo json_encode($list, JSON_PRETTY_PRINT); + + } elseif($action === 'display' && !empty($bridge)) { // DEPRECATED: 'nameBridge' scheme is replaced by 'name' in bridge parameter values // this is to keep compatibility until futher complete removal if(($pos = strpos($bridge, 'Bridge')) === (strlen($bridge) - strlen('Bridge'))) { @@ -128,143 +175,159 @@ try { define('NOPROXY', true); } - // Custom cache timeout + // Cache timeout $cache_timeout = -1; if(array_key_exists('_cache_timeout', $params)) { + if(!CUSTOM_CACHE_TIMEOUT) { throw new \HttpException('This server doesn\'t support "_cache_timeout"!'); } $cache_timeout = filter_var($params['_cache_timeout'], FILTER_VALIDATE_INT); + + } else { + $cache_timeout = $bridge->getCacheTimeout(); } + // Remove parameters that don't concern bridges + $bridge_params = array_diff_key( + $params, + array_fill_keys( + array( + 'action', + 'bridge', + 'format', + '_noproxy', + '_cache_timeout', + '_error_time' + ), '') + ); + + // Remove parameters that don't concern caches + $cache_params = array_diff_key( + $params, + array_fill_keys( + array( + 'action', + 'format', + '_noproxy', + '_cache_timeout', + '_error_time' + ), '') + ); + // Initialize cache $cache = Cache::create('FileCache'); - $cache->setPath(CACHE_DIR); + $cache->setPath(PATH_CACHE); $cache->purgeCache(86400); // 24 hours - $cache->setParameters($params); + $cache->setParameters($cache_params); - unset($params['action']); - unset($params['bridge']); - unset($params['format']); - unset($params['_noproxy']); - unset($params['_cache_timeout']); + $items = array(); + $infos = array(); + $mtime = $cache->getTime(); + + if($mtime !== false + && (time() - $cache_timeout < $mtime) + && (!defined('DEBUG') || DEBUG !== true)) { // Load cached data + + // Send "Not Modified" response if client supports it + // Implementation based on https://stackoverflow.com/a/10847262 + if(isset($_SERVER['HTTP_IF_MODIFIED_SINCE'])) { + $stime = strtotime($_SERVER['HTTP_IF_MODIFIED_SINCE']); + + if($mtime <= $stime) { // Cached data is older or same + header('Last-Modified: ' . gmdate('D, d M Y H:i:s ', $mtime) . 'GMT', true, 304); + die(); + } + } + + $cached = $cache->loadData(); + + if(isset($cached['items']) && isset($cached['extraInfos'])) { + $items = $cached['items']; + $infos = $cached['extraInfos']; + } + + } else { // Collect new data + + try { + $bridge->setDatas($bridge_params); + $bridge->collectData(); + + $items = $bridge->getItems(); + $infos = array( + 'name' => $bridge->getName(), + 'uri' => $bridge->getURI(), + 'icon' => $bridge->getIcon() + ); + } catch(Error $e) { + error_log($e); + + $item = array(); + + // Create "new" error message every 24 hours + $params['_error_time'] = urlencode((int)(time() / 86400)); + + // Error 0 is a special case (i.e. "trying to get property of non-object") + if($e->getCode() === 0) { + $item['title'] = 'Bridge encountered an unexpected situation! (' . $params['_error_time'] . ')'; + } else { + $item['title'] = 'Bridge returned error ' . $e->getCode() . '! (' . $params['_error_time'] . ')'; + } + + $item['uri'] = parse_url($_SERVER['REQUEST_URI'], PHP_URL_PATH) . '?' . http_build_query($params); + $item['timestamp'] = time(); + $item['content'] = buildBridgeException($e, $bridge); + + $items[] = $item; + } catch(Exception $e) { + error_log($e); + + $item = array(); + + // Create "new" error message every 24 hours + $params['_error_time'] = urlencode((int)(time() / 86400)); + + $item['uri'] = parse_url($_SERVER['REQUEST_URI'], PHP_URL_PATH) . '?' . http_build_query($params); + $item['title'] = 'Bridge returned error ' . $e->getCode() . '! (' . $params['_error_time'] . ')'; + $item['timestamp'] = time(); + $item['content'] = buildBridgeException($e, $bridge); + + $items[] = $item; + } + + // Store data in cache + $cache->saveData(array( + 'items' => $items, + 'extraInfos' => $infos + )); - // Load cache & data - try { - $bridge->setCache($cache); - $bridge->setCacheTimeout($cache_timeout); - $bridge->setDatas($params); - } catch(Error $e) { - http_response_code($e->getCode()); - header('Content-Type: text/html'); - die(buildBridgeException($e, $bridge)); - } catch(Exception $e) { - http_response_code($e->getCode()); - header('Content-Type: text/html'); - die(buildBridgeException($e, $bridge)); } // Data transformation try { $format = Format::create($format); - $format->setItems($bridge->getItems()); - $format->setExtraInfos($bridge->getExtraInfos()); + $format->setItems($items); + $format->setExtraInfos($infos); + $format->setLastModified($cache->getTime()); $format->display(); } catch(Error $e) { - http_response_code($e->getCode()); - header('Content-Type: text/html'); + error_log($e); + header('Content-Type: text/html', true, $e->getCode()); die(buildTransformException($e, $bridge)); } catch(Exception $e) { - http_response_code($e->getCode()); - header('Content-Type: text/html'); - die(buildBridgeException($e, $bridge)); + error_log($e); + header('Content-Type: text/html', true, $e->getCode()); + die(buildTransformException($e, $bridge)); } - - die; + } else { + echo BridgeList::create($whitelist_selection, $showInactive); } } catch(HttpException $e) { - http_response_code($e->getCode()); - header('Content-Type: text/plain'); + error_log($e); + header('Content-Type: text/plain', true, $e->getCode()); die($e->getMessage()); } catch(\Exception $e) { + error_log($e); die($e->getMessage()); } - -$formats = Format::searchInformation(); - -?> -<!DOCTYPE html> -<html lang="en"> -<head> - <meta charset="utf-8"> - <meta name="viewport" content="width=device-width, initial-scale=1.0" /> - <meta name="description" content="Rss-bridge" /> - <title>RSS-Bridge</title> - <link href="static/style.css" rel="stylesheet"> - <script src="static/search.js"></script> - <script src="static/select.js"></script> - <noscript> - <style> - .searchbar { - display: none; - } - </style> - </noscript> -</head> - -<body onload="search()"> - <?php - $status = ''; - if(defined('DEBUG') && DEBUG === true) { - $status .= 'debug mode active'; - } - - $query = filter_input(INPUT_GET, 'q'); - - echo <<<EOD - <header> - <h1>RSS-Bridge</h1> - <h2>·Reconnecting the Web·</h2> - <p class="status">{$status}</p> - </header> - <section class="searchbar"> - <h3>Search</h3> - <input type="text" name="searchfield" - id="searchfield" placeholder="Enter the bridge you want to search for" - onchange="search()" onkeyup="search()" value="{$query}"> - </section> - -EOD; - - $activeFoundBridgeCount = 0; - $showInactive = filter_input(INPUT_GET, 'show_inactive', FILTER_VALIDATE_BOOLEAN); - $inactiveBridges = ''; - $bridgeList = Bridge::listBridges(); - foreach($bridgeList as $bridgeName) { - if(Bridge::isWhitelisted($whitelist_selection, strtolower($bridgeName))) { - echo displayBridgeCard($bridgeName, $formats); - $activeFoundBridgeCount++; - } elseif($showInactive) { - // inactive bridges - $inactiveBridges .= displayBridgeCard($bridgeName, $formats, false) . PHP_EOL; - } - } - echo $inactiveBridges; - ?> - <section class="footer"> - <a href="https://github.com/RSS-Bridge/rss-bridge">RSS-Bridge ~ Public Domain</a><br /> - <p class="version"> <?= Configuration::getVersion() ?> </p> - <?= $activeFoundBridgeCount; ?>/<?= count($bridgeList) ?> active bridges. <br /> - <?php - if($activeFoundBridgeCount !== count($bridgeList)) { - // FIXME: This should be done in pure CSS - if(!$showInactive) - echo '<a href="?show_inactive=1"><button class="small">Show inactive bridges</button></a><br />'; - else - echo '<a href="?show_inactive=0"><button class="small">Hide inactive bridges</button></a><br />'; - } - ?> - </section> - </body> -</html> diff --git a/lib/Authentication.php b/lib/Authentication.php index dc75d28..da24763 100644 --- a/lib/Authentication.php +++ b/lib/Authentication.php @@ -5,8 +5,7 @@ class Authentication { if(Configuration::getConfig('authentication', 'enable') === true) { if(!Authentication::verifyPrompt()) { - header('WWW-Authenticate: Basic realm="RSS-Bridge"'); - header('HTTP/1.0 401 Unauthorized'); + header('WWW-Authenticate: Basic realm="RSS-Bridge"', true, 401); die('Please authenticate in order to access this instance !'); } diff --git a/lib/Bridge.php b/lib/Bridge.php index a2ae927..e0010df 100644 --- a/lib/Bridge.php +++ b/lib/Bridge.php @@ -1,5 +1,5 @@ <?php -require_once(__DIR__ . '/BridgeInterface.php'); + class Bridge { static protected $dirBridge; diff --git a/lib/BridgeAbstract.php b/lib/BridgeAbstract.php index 0bd1c7c..aa65411 100644 --- a/lib/BridgeAbstract.php +++ b/lib/BridgeAbstract.php @@ -1,5 +1,5 @@ <?php -require_once(__DIR__ . '/BridgeInterface.php'); + abstract class BridgeAbstract implements BridgeInterface { const NAME = 'Unnamed bridge'; @@ -9,23 +9,9 @@ abstract class BridgeAbstract implements BridgeInterface { const CACHE_TIMEOUT = 3600; const PARAMETERS = array(); - protected $cache; - protected $extraInfos; protected $items = array(); protected $inputs = array(); protected $queriedContext = ''; - protected $cacheTimeout; - - /** - * Return cachable datas (extrainfos and items) stored in the bridge - * @return mixed - */ - public function getCachable(){ - return array( - 'items' => $this->getItems(), - 'extraInfos' => $this->getExtraInfos() - ); - } /** * Return items stored in the bridge @@ -117,90 +103,37 @@ abstract class BridgeAbstract implements BridgeInterface { } /** - * Returns the name of the context matching the provided inputs - * - * @param array $inputs Associative array of inputs - * @return mixed Returns the context name or null if no match was found - */ - protected function getQueriedContext(array $inputs){ - $queriedContexts = array(); - - // Detect matching context - foreach(static::PARAMETERS as $context => $set) { - $queriedContexts[$context] = null; - - // Check if all parameters of the context are satisfied - foreach($set as $id => $properties) { - if(isset($inputs[$id]) && !empty($inputs[$id])) { - $queriedContexts[$context] = true; - } elseif(isset($properties['required']) - && $properties['required'] === true) { - $queriedContexts[$context] = false; - break; - } - } - - } - - // Abort if one of the globally required parameters is not satisfied - if(array_key_exists('global', static::PARAMETERS) - && $queriedContexts['global'] === false) { - return null; - } - unset($queriedContexts['global']); - - switch(array_sum($queriedContexts)) { - case 0: // Found no match, is there a context without parameters? - foreach($queriedContexts as $context => $queried) { - if(is_null($queried)) { - return $context; - } - } - return null; - case 1: // Found unique match - return array_search(true, $queriedContexts); - default: return false; - } - } - - /** * Defined datas with parameters depending choose bridge - * Note : you can define a cache with "setCache" * @param array array with expected bridge paramters */ public function setDatas(array $inputs){ - if(!is_null($this->cache)) { - $time = $this->cache->getTime(); - if($time !== false - && (time() - $this->getCacheTimeout() < $time) - && (!defined('DEBUG') || DEBUG !== true)) { - $cached = $this->cache->loadData(); - if(isset($cached['items']) && isset($cached['extraInfos'])) { - $this->items = $cached['items']; - $this->extraInfos = $cached['extraInfos']; - return; - } - } - } if(empty(static::PARAMETERS)) { + if(!empty($inputs)) { returnClientError('Invalid parameters value(s)'); } - $this->collectData(); - if(!is_null($this->cache)) { - $this->cache->saveData($this->getCachable()); - } return; + } - if(!validateData($inputs, static::PARAMETERS)) { - returnClientError('Invalid parameters value(s)'); + $validator = new ParameterValidator(); + + if(!$validator->validateData($inputs, static::PARAMETERS)) { + $parameters = array_map( + function($i){ return $i['name']; }, // Just display parameter names + $validator->getInvalidParameters() + ); + + returnClientError( + 'Invalid parameters value(s): ' + . implode(', ', $parameters) + ); } // Guess the paramter context from input data - $this->queriedContext = $this->getQueriedContext($inputs); + $this->queriedContext = $validator->getQueriedContext($inputs, static::PARAMETERS); if(is_null($this->queriedContext)) { returnClientError('Required parameter(s) missing'); } elseif($this->queriedContext === false) { @@ -209,11 +142,6 @@ abstract class BridgeAbstract implements BridgeInterface { $this->setInputs($inputs, $this->queriedContext); - $this->collectData(); - - if(!is_null($this->cache)) { - $this->cache->saveData($this->getCachable()); - } } /** @@ -238,48 +166,23 @@ abstract class BridgeAbstract implements BridgeInterface { } public function getName(){ - // Return cached name when bridge is using cached data - if(isset($this->extraInfos)) { - return $this->extraInfos['name']; - } - return static::NAME; } + public function getIcon(){ + return ''; + } + public function getParameters(){ return static::PARAMETERS; } public function getURI(){ - // Return cached uri when bridge is using cached data - if(isset($this->extraInfos)) { - return $this->extraInfos['uri']; - } - return static::URI; } - public function getExtraInfos(){ - return array( - 'name' => $this->getName(), - 'uri' => $this->getURI() - ); - } - - public function setCache(\CacheInterface $cache){ - $this->cache = $cache; - } - - public function setCacheTimeout($timeout){ - if(is_numeric($timeout) && ($timeout < 1 || $timeout > 86400)) { - $this->cacheTimeout = static::CACHE_TIMEOUT; - return; - } - - $this->cacheTimeout = $timeout; - } - public function getCacheTimeout(){ - return isset($this->cacheTimeout) ? $this->cacheTimeout : static::CACHE_TIMEOUT; + return static::CACHE_TIMEOUT; } + } diff --git a/lib/BridgeCard.php b/lib/BridgeCard.php new file mode 100644 index 0000000..28e74fe --- /dev/null +++ b/lib/BridgeCard.php @@ -0,0 +1,268 @@ +<?php +final class BridgeCard { + + private static function buildFormatButtons($formats) { + $buttons = ''; + + foreach($formats as $name) { + $buttons .= '<button type="submit" name="format" value="' + . $name + . '">' + . $name + . '</button>' + . PHP_EOL; + } + + return $buttons; + } + + private static function getFormHeader($bridgeName, $isHttps = false) { + $form = <<<EOD + <form method="GET" action="?"> + <input type="hidden" name="action" value="display" /> + <input type="hidden" name="bridge" value="{$bridgeName}" /> +EOD; + + if(!$isHttps) { + $form .= '<div class="secure-warning">Warning : +This bridge is not fetching its content through a secure connection</div>'; + } + + return $form; + } + + private static function getForm($bridgeName, + $formats, + $isActive = false, + $isHttps = false, + $parameterName = '', + $parameters = array()) { + $form = BridgeCard::getFormHeader($bridgeName, $isHttps); + + if(count($parameters) > 0) { + + $form .= '<div class="parameters">'; + + foreach($parameters as $id => $inputEntry) { + if(!isset($inputEntry['exampleValue'])) + $inputEntry['exampleValue'] = ''; + + if(!isset($inputEntry['defaultValue'])) + $inputEntry['defaultValue'] = ''; + + $idArg = 'arg-' + . urlencode($bridgeName) + . '-' + . urlencode($parameterName) + . '-' + . urlencode($id); + + $form .= '<label for="' + . $idArg + . '">' + . filter_var($inputEntry['name'], FILTER_SANITIZE_STRING) + . '</label>' + . PHP_EOL; + + if(!isset($inputEntry['type']) || $inputEntry['type'] === 'text') { + $form .= BridgeCard::getTextInput($inputEntry, $idArg, $id); + } elseif($inputEntry['type'] === 'number') { + $form .= BridgeCard::getNumberInput($inputEntry, $idArg, $id); + } else if($inputEntry['type'] === 'list') { + $form .= BridgeCard::getListInput($inputEntry, $idArg, $id); + } elseif($inputEntry['type'] === 'checkbox') { + $form .= BridgeCard::getCheckboxInput($inputEntry, $idArg, $id); + } + } + + $form .= '</div>'; + + } + + if($isActive) { + $form .= BridgeCard::buildFormatButtons($formats); + } else { + $form .= '<span style="font-weight: bold;">Inactive</span>'; + } + + return $form . '</form>' . PHP_EOL; + } + + private static function getInputAttributes($entry) { + $retVal = ''; + + if(isset($entry['required']) && $entry['required'] === true) + $retVal .= ' required'; + + if(isset($entry['pattern'])) + $retVal .= ' pattern="' . $entry['pattern'] . '"'; + + if(isset($entry['title'])) + $retVal .= ' title="' . filter_var($entry['title'], FILTER_SANITIZE_STRING) . '"'; + + return $retVal; + } + + private static function getTextInput($entry, $id, $name) { + return '<input ' + . BridgeCard::getInputAttributes($entry) + . ' id="' + . $id + . '" type="text" value="' + . filter_var($entry['defaultValue'], FILTER_SANITIZE_STRING) + . '" placeholder="' + . filter_var($entry['exampleValue'], FILTER_SANITIZE_STRING) + . '" name="' + . $name + . '" />' + . PHP_EOL; + } + + private static function getNumberInput($entry, $id, $name) { + return '<input ' + . BridgeCard::getInputAttributes($entry) + . ' id="' + . $id + . '" type="number" value="' + . filter_var($entry['defaultValue'], FILTER_SANITIZE_NUMBER_INT) + . '" placeholder="' + . filter_var($entry['exampleValue'], FILTER_SANITIZE_NUMBER_INT) + . '" name="' + . $name + . '" />' + . PHP_EOL; + } + + private static function getListInput($entry, $id, $name) { + $list = '<select ' + . BridgeCard::getInputAttributes($entry) + . ' id="' + . $id + . '" name="' + . $name + . '" >'; + + foreach($entry['values'] as $name => $value) { + if(is_array($value)) { + $list .= '<optgroup label="' . htmlentities($name) . '">'; + foreach($value as $subname => $subvalue) { + if($entry['defaultValue'] === $subname + || $entry['defaultValue'] === $subvalue) { + $list .= '<option value="' + . $subvalue + . '" selected>' + . $subname + . '</option>'; + } else { + $list .= '<option value="' + . $subvalue + . '">' + . $subname + . '</option>'; + } + } + $list .= '</optgroup>'; + } else { + if($entry['defaultValue'] === $name + || $entry['defaultValue'] === $value) { + $list .= '<option value="' + . $value + . '" selected>' + . $name + . '</option>'; + } else { + $list .= '<option value="' + . $value + . '">' + . $name + . '</option>'; + } + } + } + + $list .= '</select>'; + + return $list; + } + + private static function getCheckboxInput($entry, $id, $name) { + return '<input ' + . BridgeCard::getInputAttributes($entry) + . ' id="' + . $id + . '" type="checkbox" name="' + . $name + . '" ' + . ($entry['defaultValue'] === 'checked' ? 'checked' : '') + . ' />' + . PHP_EOL; + } + + static function displayBridgeCard($bridgeName, $formats, $isActive = true){ + + $bridge = Bridge::create($bridgeName); + + if($bridge == false) + return ''; + + $isHttps = strpos($bridge->getURI(), 'https') === 0; + + $uri = $bridge->getURI(); + $name = $bridge->getName(); + $icon = $bridge->getIcon(); + $description = $bridge->getDescription(); + $parameters = $bridge->getParameters(); + + if(defined('PROXY_URL') && PROXY_BYBRIDGE) { + $parameters['global']['_noproxy'] = array( + 'name' => 'Disable proxy (' . ((defined('PROXY_NAME') && PROXY_NAME) ? PROXY_NAME : PROXY_URL) . ')', + 'type' => 'checkbox' + ); + } + + if(CUSTOM_CACHE_TIMEOUT) { + $parameters['global']['_cache_timeout'] = array( + 'name' => 'Cache timeout in seconds', + 'type' => 'number', + 'defaultValue' => $bridge->getCacheTimeout() + ); + } + + $card = <<<CARD + <section id="bridge-{$bridgeName}" data-ref="{$bridgeName}"> + <h2><a href="{$uri}">{$name}</a></h2> + <p class="description">{$description}</p> + <input type="checkbox" class="showmore-box" id="showmore-{$bridgeName}" /> + <label class="showmore" for="showmore-{$bridgeName}">Show more</label> +CARD; + + // If we don't have any parameter for the bridge, we print a generic form to load it. + if(count($parameters) === 0 + || count($parameters) === 1 && array_key_exists('global', $parameters)) { + + $card .= BridgeCard::getForm($bridgeName, $formats, $isActive, $isHttps); + + } else { + + foreach($parameters as $parameterName => $parameter) { + if(!is_numeric($parameterName) && $parameterName === 'global') + continue; + + if(array_key_exists('global', $parameters)) + $parameter = array_merge($parameter, $parameters['global']); + + if(!is_numeric($parameterName)) + $card .= '<h5>' . $parameterName . '</h5>' . PHP_EOL; + + $card .= BridgeCard::getForm($bridgeName, $formats, $isActive, $isHttps, $parameterName, $parameter); + } + + } + + $card .= '<label class="showless" for="showmore-' . $bridgeName . '">Show less</label>'; + $card .= '<p class="maintainer">' . $bridge->getMaintainer() . '</p>'; + $card .= '</section>'; + + return $card; + } +} diff --git a/lib/BridgeInterface.php b/lib/BridgeInterface.php index b8f5cf4..f2ff11d 100644 --- a/lib/BridgeInterface.php +++ b/lib/BridgeInterface.php @@ -7,13 +7,6 @@ interface BridgeInterface { public function collectData(); /** - * Returns an array of cachable elements - * - * @return array Associative array of cachable elements - */ - public function getCachable(); - - /** * Returns the description * * @return string Description @@ -21,13 +14,6 @@ interface BridgeInterface { public function getDescription(); /** - * Return an array of extra information - * - * @return array Associative array of extra information - */ - public function getExtraInfos(); - - /** * Returns an array of collected items * * @return array Associative array of items @@ -49,6 +35,13 @@ interface BridgeInterface { public function getName(); /** + * Returns the bridge icon + * + * @return string Bridge icon + */ + public function getIcon(); + + /** * Returns the bridge parameters * * @return array Bridge parameters @@ -63,22 +56,6 @@ interface BridgeInterface { public function getURI(); /** - * Sets the cache instance - * - * @param object CacheInterface The cache instance - */ - public function setCache(\CacheInterface $cache); - - /** - * Sets the timeout for clearing the cache files. The timeout must be - * specified between 1..86400 seconds (max. 24 hours). The default timeout - * (specified by the bridge maintainer) applies for invalid values. - * - * @param int $timeout The cache timeout in seconds - */ - public function setCacheTimeout($timeout); - - /** * Returns the cache timeout * * @return int Cache timeout diff --git a/lib/BridgeList.php b/lib/BridgeList.php new file mode 100644 index 0000000..4c59f10 --- /dev/null +++ b/lib/BridgeList.php @@ -0,0 +1,149 @@ +<?php +final class BridgeList { + + private static function getHead() { + return <<<EOD +<head> + <meta charset="utf-8"> + <meta name="viewport" content="width=device-width, initial-scale=1.0" /> + <meta name="description" content="RSS-Bridge" /> + <title>RSS-Bridge</title> + <link href="static/style.css" rel="stylesheet"> + <script src="static/search.js"></script> + <script src="static/select.js"></script> + <noscript> + <style> + .searchbar { + display: none; + } + </style> + </noscript> +</head> +EOD; + } + + private static function getBridges($whitelist, $showInactive, &$totalBridges, &$totalActiveBridges) { + + $body = ''; + $totalActiveBridges = 0; + $inactiveBridges = ''; + + $bridgeList = Bridge::listBridges(); + $formats = Format::searchInformation(); + + $totalBridges = count($bridgeList); + + foreach($bridgeList as $bridgeName) { + + if(Bridge::isWhitelisted($whitelist, strtolower($bridgeName))) { + + $body .= BridgeCard::displayBridgeCard($bridgeName, $formats); + $totalActiveBridges++; + + } elseif($showInactive) { + + // inactive bridges + $inactiveBridges .= BridgeCard::displayBridgeCard($bridgeName, $formats, false) . PHP_EOL; + + } + + } + + $body .= $inactiveBridges; + + return $body; + } + + private static function getHeader() { + $warning = ''; + + if(defined('DEBUG') && DEBUG === true) { + if(defined('DEBUG_INSECURE') && DEBUG_INSECURE === true) { + $warning .= <<<EOD +<section class="critical-warning">Warning : Debug mode is active from any location, +make sure only you can access RSS-Bridge.</section> +EOD; + } else { + $warning .= <<<EOD +<section class="warning">Warning : Debug mode is active from your IP address, +your requests will bypass the cache.</section> +EOD; + } + } + + return <<<EOD +<header> + <h1>RSS-Bridge</h1> + <h2>Reconnecting the Web</h2> + {$warning} +</header> +EOD; + } + + private static function getSearchbar() { + $query = filter_input(INPUT_GET, 'q'); + + return <<<EOD +<section class="searchbar"> + <h3>Search</h3> + <input type="text" name="searchfield" + id="searchfield" placeholder="Enter the bridge you want to search for" + onchange="search()" onkeyup="search()" value="{$query}"> +</section> +EOD; + } + + private static function getFooter($totalBridges, $totalActiveBridges, $showInactive) { + $version = Configuration::getVersion(); + + $email = Configuration::getConfig('admin', 'email'); + $admininfo = ''; + if (!empty($email)) { + $admininfo = <<<EOD +<br /> +<span> + You may email the administrator of this RSS-Bridge instance + at <a href="mailto:{$email}">{$email}</a> +</span> +EOD; + } + + $inactive = ''; + + if($totalActiveBridges !== $totalBridges) { + + if(!$showInactive) { + $inactive = '<a href="?show_inactive=1"><button class="small">Show inactive bridges</button></a><br>'; + } else { + $inactive = '<a href="?show_inactive=0"><button class="small">Hide inactive bridges</button></a><br>'; + } + + } + + return <<<EOD +<section class="footer"> + <a href="https://github.com/rss-bridge/rss-bridge">RSS-Bridge ~ Public Domain</a><br> + <p class="version">{$version}</p> + {$totalActiveBridges}/{$totalBridges} active bridges.<br> + {$inactive} + {$admininfo} +</section> +EOD; + } + + static function create($whitelist, $showInactive = true) { + + $totalBridges = 0; + $totalActiveBridges = 0; + + return '<!DOCTYPE html><html lang="en">' + . BridgeList::getHead() + . '<body onload="search()">' + . BridgeList::getHeader() + . BridgeList::getSearchbar() + . BridgeList::getBridges($whitelist, $showInactive, $totalBridges, $totalActiveBridges) + . BridgeList::getFooter($totalBridges, $totalActiveBridges, $showInactive) + . '</body></html>'; + + } +} diff --git a/lib/Cache.php b/lib/Cache.php index bc17758..88be5a1 100644 --- a/lib/Cache.php +++ b/lib/Cache.php @@ -1,5 +1,5 @@ <?php -require_once(__DIR__ . '/CacheInterface.php'); + class Cache { static protected $dirCache; diff --git a/lib/Configuration.php b/lib/Configuration.php index 620b0e4..3002822 100644 --- a/lib/Configuration.php +++ b/lib/Configuration.php @@ -1,15 +1,15 @@ <?php class Configuration { - public static $VERSION = '2018-07-17'; + public static $VERSION = '2018-11-10'; public static $config = null; public static function verifyInstallation() { // Check PHP version - if(version_compare(PHP_VERSION, PHP_VERSION_REQUIRED) === -1) - die('RSS-Bridge requires at least PHP version ' . PHP_VERSION_REQUIRED . '!'); + if(version_compare(PHP_VERSION, '5.6.0') === -1) + die('RSS-Bridge requires at least PHP version 5.6.0!'); // extensions check if(!extension_loaded('openssl')) @@ -27,9 +27,12 @@ class Configuration { if(!extension_loaded('curl')) die('"curl" extension not loaded. Please check "php.ini"'); + if(!extension_loaded('json')) + die('"json" extension not loaded. Please check "php.ini"'); + // Check cache folder permissions (write permissions required) - if(!is_writable(CACHE_DIR)) - die('RSS-Bridge does not have write permissions for ' . CACHE_DIR . '!'); + if(!is_writable(PATH_CACHE)) + die('RSS-Bridge does not have write permissions for ' . PATH_CACHE . '!'); // Check whitelist file permissions (only in DEBUG mode) if(!file_exists(WHITELIST_FILE) && !is_writable(dirname(WHITELIST_FILE))) @@ -104,7 +107,8 @@ class Configuration { $headFile = '.git/HEAD'; - if(file_exists($headFile)) { + // '@' is used to mute open_basedir warning + if(@is_readable($headFile)) { $revisionHashFile = '.git/' . substr(file_get_contents($headFile), 5, -1); $branchName = explode('/', $revisionHashFile)[3]; diff --git a/lib/Exceptions.php b/lib/Exceptions.php index 9b89320..32b33f2 100644 --- a/lib/Exceptions.php +++ b/lib/Exceptions.php @@ -19,7 +19,8 @@ function buildGitHubIssueQuery($title, $body, $labels = null, $maintainer = null } // Add title and body - $uri = 'https://github.com/rss-bridge/rss-bridge/issues/new?title=' + $uri = REPOSITORY + . 'issues/new?title=' . urlencode($title) . '&body=' . urlencode($body); @@ -69,14 +70,18 @@ function buildBridgeException($e, $bridge){ . Configuration::getVersion() . '`'; + $body_html = nl2br($body); $link = buildGitHubIssueQuery($title, $body, 'bug report', $bridge->getMaintainer()); $header = buildHeader($e, $bridge); - $message = "<strong>{$bridge->getName()}</strong> was -unable to receive or process the remote website's content!"; + $message = <<<EOD +<strong>{$bridge->getName()}</strong> was unable to receive or process the +remote website's content!<br> +{$body_html} +EOD; $section = buildSection($e, $bridge, $message, $link); - return buildPage($title, $header, $section); + return $section; } /** @@ -127,7 +132,7 @@ function buildSection($e, $bridge, $message, $link){ <ul class="advice"> <li>Press Return to check your input parameters</li> <li>Press F5 to retry</li> - <li>Open a GitHub Issue if this error persists</li> + <li>Open a <a href="{$link}">GitHub Issue</a> if this error persists</li> </ul> </div> <a href="{$link}" title="After clicking this button you can review diff --git a/lib/FeedExpander.php b/lib/FeedExpander.php index 6e1f16f..269de49 100644 --- a/lib/FeedExpander.php +++ b/lib/FeedExpander.php @@ -1,5 +1,5 @@ <?php -require_once(__DIR__ . '/BridgeInterface.php'); + abstract class FeedExpander extends BridgeAbstract { private $name; @@ -115,12 +115,27 @@ abstract class FeedExpander extends BridgeAbstract { } protected function parseATOMItem($feedItem){ - $item = array(); + // Some ATOM entries also contain RSS 2.0 fields + $item = $this->parseRSS_2_0_Item($feedItem); + if(isset($feedItem->id)) $item['uri'] = (string)$feedItem->id; if(isset($feedItem->title)) $item['title'] = (string)$feedItem->title; if(isset($feedItem->updated)) $item['timestamp'] = strtotime((string)$feedItem->updated); if(isset($feedItem->author)) $item['author'] = (string)$feedItem->author->name; if(isset($feedItem->content)) $item['content'] = (string)$feedItem->content; + + //When "link" field is present, URL is more reliable than "id" field + if (count($feedItem->link) === 1) { + $this->uri = (string)$feedItem->link[0]['href']; + } else { + foreach($feedItem->link as $link) { + if(strtolower($link['rel']) === 'alternate') { + $item['uri'] = (string)$link['href']; + break; + } + } + } + return $item; } @@ -130,6 +145,7 @@ abstract class FeedExpander extends BridgeAbstract { if(isset($feedItem->title)) $item['title'] = (string)$feedItem->title; // rss 0.91 doesn't support timestamps // rss 0.91 doesn't support authors + // rss 0.91 doesn't support enclosures if(isset($feedItem->description)) $item['content'] = (string)$feedItem->description; return $item; } @@ -154,11 +170,17 @@ abstract class FeedExpander extends BridgeAbstract { $namespaces = $feedItem->getNamespaces(true); if(isset($namespaces['dc'])) $dc = $feedItem->children($namespaces['dc']); + if(isset($namespaces['media'])) $media = $feedItem->children($namespaces['media']); if(isset($feedItem->guid)) { foreach($feedItem->guid->attributes() as $attribute => $value) { if($attribute === 'isPermaLink' - && ($value === 'true' || filter_var($feedItem->guid, FILTER_VALIDATE_URL))) { + && ($value === 'true' || ( + filter_var($feedItem->guid, FILTER_VALIDATE_URL) + && !filter_var($item['uri'], FILTER_VALIDATE_URL) + ) + ) + ) { $item['uri'] = (string)$feedItem->guid; break; } @@ -170,11 +192,21 @@ abstract class FeedExpander extends BridgeAbstract { } elseif(isset($dc->date)) { $item['timestamp'] = strtotime((string)$dc->date); } + if(isset($feedItem->author)) { $item['author'] = (string)$feedItem->author; + } elseif (isset($feedItem->creator)) { + $item['author'] = (string)$feedItem->creator; } elseif(isset($dc->creator)) { $item['author'] = (string)$dc->creator; + } elseif(isset($media->credit)) { + $item['author'] = (string)$media->credit; + } + + if(isset($feedItem->enclosure) && !empty($feedItem->enclosure['url'])) { + $item['enclosures'] = array((string)$feedItem->enclosure['url']); } + return $item; } @@ -199,10 +231,14 @@ abstract class FeedExpander extends BridgeAbstract { } public function getURI(){ - return $this->uri ?: parent::getURI(); + return !empty($this->uri) ? $this->uri : parent::getURI(); } public function getName(){ - return $this->name ?: parent::getName(); + return !empty($this->name) ? $this->name : parent::getName(); + } + + public function getIcon(){ + return !empty($this->icon) ? $this->icon : parent::getIcon(); } } diff --git a/lib/Format.php b/lib/Format.php index ab932cc..d0e4e72 100644 --- a/lib/Format.php +++ b/lib/Format.php @@ -1,5 +1,5 @@ <?php -require_once(__DIR__ . '/FormatInterface.php'); + class Format { static protected $dirFormat; diff --git a/lib/FormatAbstract.php b/lib/FormatAbstract.php index 46d5e4a..8fb642c 100644 --- a/lib/FormatAbstract.php +++ b/lib/FormatAbstract.php @@ -1,5 +1,5 @@ <?php -require_once(__DIR__ . '/FormatInterface.php'); + abstract class FormatAbstract implements FormatInterface { const DEFAULT_CHARSET = 'UTF-8'; @@ -7,6 +7,7 @@ abstract class FormatAbstract implements FormatInterface { $contentType, $charset, $items, + $lastModified, $extraInfos; public function setCharset($charset){ @@ -27,11 +28,18 @@ abstract class FormatAbstract implements FormatInterface { return $this; } + public function setLastModified($lastModified){ + $this->lastModified = $lastModified; + } + protected function callContentType(){ header('Content-Type: ' . $this->contentType); } public function display(){ + if ($this->lastModified) { + header('Last-Modified: ' . gmdate('D, d M Y H:i:s ', $this->lastModified) . 'GMT'); + } echo $this->stringify(); return $this; @@ -51,12 +59,12 @@ abstract class FormatAbstract implements FormatInterface { } /** - * Define common informations can be required by formats and set default value for unknow values + * Define common informations can be required by formats and set default value for unknown values * @param array $extraInfos array with know informations (there isn't merge !!!) * @return this */ public function setExtraInfos(array $extraInfos = array()){ - foreach(array('name', 'uri') as $infoName) { + foreach(array('name', 'uri', 'icon') as $infoName) { if(!isset($extraInfos[$infoName])) { $extraInfos[$infoName] = ''; } diff --git a/lib/ParameterValidator.php b/lib/ParameterValidator.php new file mode 100644 index 0000000..c278e4d --- /dev/null +++ b/lib/ParameterValidator.php @@ -0,0 +1,171 @@ +<?php +/** + * Implements a validator for bridge parameters + */ +class ParameterValidator { + private $invalid = array(); + + private function addInvalidParameter($name, $reason){ + $this->invalid[] = array( + 'name' => $name, + 'reason' => $reason + ); + } + + /** + * Returns an array of invalid parameters, where each element is an + * array of 'name' and 'reason'. + */ + public function getInvalidParameters() { + return $this->invalid; + } + + private function validateTextValue($value, $pattern = null){ + if(!is_null($pattern)) { + $filteredValue = filter_var($value, + FILTER_VALIDATE_REGEXP, + array('options' => array( + 'regexp' => '/^' . $pattern . '$/' + ) + )); + } else { + $filteredValue = filter_var($value); + } + + if($filteredValue === false) + return null; + + return $filteredValue; + } + + private function validateNumberValue($value){ + $filteredValue = filter_var($value, FILTER_VALIDATE_INT); + + if($filteredValue === false) + return null; + + return $filteredValue; + } + + private function validateCheckboxValue($value){ + return filter_var($value, FILTER_VALIDATE_BOOLEAN, FILTER_NULL_ON_FAILURE); + } + + private function validateListValue($value, $expectedValues){ + $filteredValue = filter_var($value); + + if($filteredValue === false) + return null; + + if(!in_array($filteredValue, $expectedValues)) { // Check sub-values? + foreach($expectedValues as $subName => $subValue) { + if(is_array($subValue) && in_array($filteredValue, $subValue)) + return $filteredValue; + } + return null; + } + + return $filteredValue; + } + + /** + * Checks if all required parameters are supplied by the user + * @param $data An array of parameters provided by the user + * @param $parameters An array of bridge parameters + */ + public function validateData(&$data, $parameters){ + + if(!is_array($data)) + return false; + + foreach($data as $name => $value) { + $registered = false; + foreach($parameters as $context => $set) { + if(array_key_exists($name, $set)) { + $registered = true; + if(!isset($set[$name]['type'])) { + $set[$name]['type'] = 'text'; + } + + switch($set[$name]['type']) { + case 'number': + $data[$name] = $this->validateNumberValue($value); + break; + case 'checkbox': + $data[$name] = $this->validateCheckboxValue($value); + break; + case 'list': + $data[$name] = $this->validateListValue($value, $set[$name]['values']); + break; + default: + case 'text': + if(isset($set[$name]['pattern'])) { + $data[$name] = $this->validateTextValue($value, $set[$name]['pattern']); + } else { + $data[$name] = $this->validateTextValue($value); + } + break; + } + + if(is_null($data[$name]) && isset($set[$name]['required']) && $set[$name]['required']) { + $this->addInvalidParameter($name, 'Parameter is invalid!'); + } + } + } + + if(!$registered) { + $this->addInvalidParameter($name, 'Parameter is not registered!'); + } + } + + return empty($this->invalid); + } + + /** + * Returns the name of the context matching the provided inputs + * + * @param array $data Associative array of user data + * @param array $parameters Array of bridge parameters + * @return mixed Returns the context name or null if no match was found + */ + public function getQueriedContext($data, $parameters){ + $queriedContexts = array(); + + // Detect matching context + foreach($parameters as $context => $set) { + $queriedContexts[$context] = null; + + // Check if all parameters of the context are satisfied + foreach($set as $id => $properties) { + if(isset($data[$id]) && !empty($data[$id])) { + $queriedContexts[$context] = true; + } elseif(isset($properties['required']) + && $properties['required'] === true) { + $queriedContexts[$context] = false; + break; + } + } + + } + + // Abort if one of the globally required parameters is not satisfied + if(array_key_exists('global', $parameters) + && $queriedContexts['global'] === false) { + return null; + } + unset($queriedContexts['global']); + + switch(array_sum($queriedContexts)) { + case 0: // Found no match, is there a context without parameters? + foreach($queriedContexts as $context => $queried) { + if(is_null($queried)) { + return $context; + } + } + return null; + case 1: // Found unique match + return array_search(true, $queriedContexts); + default: return false; + } + } +} diff --git a/lib/RssBridge.php b/lib/RssBridge.php index 8d0ef90..c0ab675 100644 --- a/lib/RssBridge.php +++ b/lib/RssBridge.php @@ -1,55 +1,37 @@ <?php -/* rss-bridge library. -Foundation functions for rss-bridge project. -See https://github.com/sebsauvage/rss-bridge -Licence: Public domain. -*/ -define('PATH_VENDOR', '/../vendor'); - -require __DIR__ . '/Exceptions.php'; -require __DIR__ . '/Format.php'; -require __DIR__ . '/FormatAbstract.php'; -require __DIR__ . '/Bridge.php'; -require __DIR__ . '/BridgeAbstract.php'; -require __DIR__ . '/FeedExpander.php'; -require __DIR__ . '/Cache.php'; -require __DIR__ . '/Authentication.php'; -require __DIR__ . '/Configuration.php'; - -require __DIR__ . '/validation.php'; -require __DIR__ . '/html.php'; -require __DIR__ . '/error.php'; -require __DIR__ . '/contents.php'; - -$vendorLibSimpleHtmlDom = __DIR__ . PATH_VENDOR . '/simplehtmldom/simple_html_dom.php'; -if(!file_exists($vendorLibSimpleHtmlDom)) { - throw new \HttpException('"PHP Simple HTML DOM Parser" library is missing. - Get it from http://simplehtmldom.sourceforge.net and place the script "simple_html_dom.php" in ' - . substr(PATH_VENDOR, 4) - . '/simplehtmldom/', - 500); -} -require_once $vendorLibSimpleHtmlDom; - -/* Example use - - require_once __DIR__ . '/lib/RssBridge.php'; - - // Data retrieval - Bridge::setDir(__DIR__ . '/bridges/'); - $bridge = Bridge::create('GoogleSearch'); - $bridge->collectData($_REQUEST); - - // Data transformation - Format::setDir(__DIR__ . '/formats/'); - $format = Format::create('Atom'); - $format - ->setItems($bridge->getItems()) - ->setExtraInfos(array( - 'name' => $bridge->getName(), - 'uri' => $bridge->getURI(), - )) - ->display(); - -*/ +define('PATH_LIB', __DIR__ . '/../lib/'); // Path to core library +define('PATH_LIB_VENDOR', __DIR__ . '/../vendor/'); // Path to vendor library +define('PATH_LIB_BRIDGES', __DIR__ . '/../bridges/'); // Path to bridges library +define('PATH_LIB_FORMATS', __DIR__ . '/../formats/'); // Path to formats library +define('PATH_LIB_CACHES', __DIR__ . '/../caches/'); // Path to caches library +define('PATH_CACHE', __DIR__ . '/../cache'); // Path to cache folder +define('REPOSITORY', 'https://github.com/RSS-Bridge/rss-bridge/'); + +// Interfaces +require_once PATH_LIB . 'BridgeInterface.php'; +require_once PATH_LIB . 'CacheInterface.php'; +require_once PATH_LIB . 'FormatInterface.php'; + +// Classes +require_once PATH_LIB . 'Exceptions.php'; +require_once PATH_LIB . 'Format.php'; +require_once PATH_LIB . 'FormatAbstract.php'; +require_once PATH_LIB . 'Bridge.php'; +require_once PATH_LIB . 'BridgeAbstract.php'; +require_once PATH_LIB . 'FeedExpander.php'; +require_once PATH_LIB . 'Cache.php'; +require_once PATH_LIB . 'Authentication.php'; +require_once PATH_LIB . 'Configuration.php'; +require_once PATH_LIB . 'BridgeCard.php'; +require_once PATH_LIB . 'BridgeList.php'; +require_once PATH_LIB . 'ParameterValidator.php'; + +// Functions +require_once PATH_LIB . 'html.php'; +require_once PATH_LIB . 'error.php'; +require_once PATH_LIB . 'contents.php'; + +// Vendor +require_once PATH_LIB_VENDOR . 'simplehtmldom/simple_html_dom.php'; +require_once PATH_LIB_VENDOR . 'php-urljoin/src/urljoin.php'; diff --git a/lib/contents.php b/lib/contents.php index 416fb7d..3f3d36c 100644 --- a/lib/contents.php +++ b/lib/contents.php @@ -1,35 +1,77 @@ <?php function getContents($url, $header = array(), $opts = array()){ + debugMessage('Reading contents from "' . $url . '"'); + $ch = curl_init($url); curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true); - if(is_array($header) && count($header) !== 0) + if(is_array($header) && count($header) !== 0) { + + debugMessage('Setting headers: ' . json_encode($header)); curl_setopt($ch, CURLOPT_HTTPHEADER, $header); + } + curl_setopt($ch, CURLOPT_USERAGENT, ini_get('user_agent')); curl_setopt($ch, CURLOPT_ENCODING, ''); curl_setopt($ch, CURLOPT_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS); - if(is_array($opts)) { + if(is_array($opts) && count($opts) !== 0) { + + debugMessage('Setting options: ' . json_encode($opts)); + foreach($opts as $key => $value) { curl_setopt($ch, $key, $value); } + } if(defined('PROXY_URL') && !defined('NOPROXY')) { + + debugMessage('Setting proxy url: ' . PROXY_URL); curl_setopt($ch, CURLOPT_PROXY, PROXY_URL); + } - $content = curl_exec($ch); + // We always want the response header as part of the data! + curl_setopt($ch, CURLOPT_HEADER, true); + + $data = curl_exec($ch); $curlError = curl_error($ch); $curlErrno = curl_errno($ch); - curl_close($ch); - if($content === false) + if($data === false) debugMessage('Cant\'t download ' . $url . ' cUrl error: ' . $curlError . ' (' . $curlErrno . ')'); - return $content; + $headerSize = curl_getinfo($ch, CURLINFO_HEADER_SIZE); + $errorCode = curl_getinfo($ch, CURLINFO_HTTP_CODE); + $header = substr($data, 0, $headerSize); + + debugMessage('Response header: ' . $header); + + $headers = parseResponseHeader($header); + $finalHeader = end($headers); + + if($errorCode !== 200) { + + if(array_key_exists('Server', $finalHeader) && strpos($finalHeader['Server'], 'cloudflare') !== false) { + returnServerError(<<< EOD +The server responded with a Cloudflare challenge, which is not supported by RSS-Bridge! +If this error persists longer than a week, please consider opening an issue on GitHub! +EOD + ); + } + + returnError(<<<EOD +The requested resource cannot be found! +Please make sure your input parameters are correct! +EOD + , $errorCode); + } + + curl_close($ch); + return substr($data, $headerSize); } function getSimpleHTMLDOM($url, @@ -71,7 +113,7 @@ $defaultSpanText = DEFAULT_SPAN_TEXT){ // Initialize cache $cache = Cache::create('FileCache'); - $cache->setPath(CACHE_DIR . '/pages'); + $cache->setPath(PATH_CACHE . '/pages'); $cache->purgeCache(86400); // 24 hours (forced) $params = [$url]; @@ -98,3 +140,90 @@ $defaultSpanText = DEFAULT_SPAN_TEXT){ $defaultBRText, $defaultSpanText); } + +/** + * Parses the provided response header into an associative array + * + * Based on https://stackoverflow.com/a/18682872 + */ +function parseResponseHeader($header) { + + $headers = array(); + $requests = explode("\r\n\r\n", trim($header)); + + foreach ($requests as $request) { + + $header = array(); + + foreach (explode("\r\n", $request) as $i => $line) { + + if($i === 0) { + $header['http_code'] = $line; + } else { + + list ($key, $value) = explode(': ', $line); + $header[$key] = $value; + + } + + } + + $headers[] = $header; + + } + + return $headers; + +} + +/** + * Determine MIME type from URL/Path file extension + * Remark: Built-in functions mime_content_type or fileinfo requires fetching remote content + * Remark: A bridge can hint for a MIME type by appending #.ext to a URL, e.g. #.image + * Based on https://stackoverflow.com/a/1147952 + */ +function getMimeType($url) { + static $mime = null; + + if (is_null($mime)) { + // Default values, overriden by /etc/mime.types when present + $mime = array( + 'jpg' => 'image/jpeg', + 'gif' => 'image/gif', + 'png' => 'image/png', + 'image' => 'image/*' + ); + // '@' is used to mute open_basedir warning, see issue #818 + if (@is_readable('/etc/mime.types')) { + $file = fopen('/etc/mime.types', 'r'); + while(($line = fgets($file)) !== false) { + $line = trim(preg_replace('/#.*/', '', $line)); + if(!$line) + continue; + $parts = preg_split('/\s+/', $line); + if(count($parts) == 1) + continue; + $type = array_shift($parts); + foreach($parts as $part) + $mime[$part] = $type; + } + fclose($file); + } + } + + if (strpos($url, '?') !== false) { + $url_temp = substr($url, 0, strpos($url, '?')); + if (strpos($url, '#') !== false) { + $anchor = substr($url, strpos($url, '#')); + $url_temp .= $anchor; + } + $url = $url_temp; + } + + $ext = strtolower(pathinfo($url, PATHINFO_EXTENSION)); + if (!empty($mime[$ext])) { + return $mime[$ext]; + } + + return 'application/octet-stream'; +} diff --git a/lib/error.php b/lib/error.php index ae18f6f..34ad9dd 100644 --- a/lib/error.php +++ b/lib/error.php @@ -20,7 +20,7 @@ function debugMessage($text){ $calling = $backtrace[2]; $message = $calling['file'] . ':' . $calling['line'] . ' class ' - . $calling['class'] . '->' + . (isset($calling['class']) ? $calling['class'] : '<no-class>') . '->' . $calling['function'] . ' - ' . $text; diff --git a/lib/html.php b/lib/html.php index 297ab80..f87991f 100644 --- a/lib/html.php +++ b/lib/html.php @@ -1,304 +1,4 @@ <?php -function displayBridgeCard($bridgeName, $formats, $isActive = true){ - - $getHelperButtonsFormat = function($formats){ - $buttons = ''; - foreach($formats as $name) { - $buttons .= '<button type="submit" name="format" value="' - . $name - . '">' - . $name - . '</button>' - . PHP_EOL; - } - - return $buttons; - }; - - $getFormHeader = function($bridgeName){ - return <<<EOD - <form method="GET" action="?"> - <input type="hidden" name="action" value="display" /> - <input type="hidden" name="bridge" value="{$bridgeName}" /> -EOD; - }; - - $bridge = Bridge::create($bridgeName); - - if($bridge == false) - return ''; - - $HTTPSWarning = ''; - if(strpos($bridge->getURI(), 'https') !== 0) { - - $HTTPSWarning = '<div class="secure-warning">Warning : - This bridge is not fetching its content through a secure connection</div>'; - - } - - $name = '<a href="' . $bridge->getURI() . '">' . $bridge->getName() . '</a>'; - $description = $bridge->getDescription(); - - $card = <<<CARD - <section id="bridge-{$bridgeName}" data-ref="{$bridgeName}"> - <h2>{$name}</h2> - <p class="description"> - {$description} - </p> - <input type="checkbox" class="showmore-box" id="showmore-{$bridgeName}" /> - <label class="showmore" for="showmore-{$bridgeName}">Show more</label> -CARD; - - // If we don't have any parameter for the bridge, we print a generic form to load it. - if(count($bridge->getParameters()) == 0) { - - $card .= $getFormHeader($bridgeName); - $card .= $HTTPSWarning; - - if($isActive) { - if(defined('PROXY_URL') && PROXY_BYBRIDGE) { - $idArg = 'arg-' - . urlencode($bridgeName) - . '-' - . urlencode('proxyoff') - . '-' - . urlencode('_noproxy'); - - $card .= '<input id="' - . $idArg - . '" type="checkbox" name="_noproxy" />' - . PHP_EOL; - - $card .= '<label for="' - . $idArg - . '">Disable proxy (' - . ((defined('PROXY_NAME') && PROXY_NAME) ? PROXY_NAME : PROXY_URL) - . ')</label><br />' - . PHP_EOL; - } if(CUSTOM_CACHE_TIMEOUT) { - $idArg = 'arg-' - . urlencode($bridgeName) - . '-' - . urlencode('_cache_timeout'); - - $card .= '<label for="' - . $idArg - . '">Cache timeout in seconds : </label>' - . PHP_EOL; - - $card .= '<input id="' - . $idArg - . '" type="number" value="' - . $bridge->getCacheTimeout() - . '" name="_cache_timeout" /><br />' - . PHP_EOL; - } - $card .= $getHelperButtonsFormat($formats); - } else { - $card .= '<span style="font-weight: bold;">Inactive</span>'; - } - - $card .= '</form>' . PHP_EOL; - } - - $hasGlobalParameter = array_key_exists('global', $bridge->getParameters()); - - if($hasGlobalParameter) { - $globalParameters = $bridge->getParameters()['global']; - } - - foreach($bridge->getParameters() as $parameterName => $parameter) { - if(!is_numeric($parameterName) && $parameterName == 'global') - continue; - - if($hasGlobalParameter) - $parameter = array_merge($parameter, $globalParameters); - - if(!is_numeric($parameterName)) - $card .= '<h5>' . $parameterName . '</h5>' . PHP_EOL; - - $card .= $getFormHeader($bridgeName); - $card .= $HTTPSWarning; - - foreach($parameter as $id => $inputEntry) { - $additionalInfoString = ''; - - if(isset($inputEntry['required']) && $inputEntry['required'] === true) - $additionalInfoString .= ' required'; - - if(isset($inputEntry['pattern'])) - $additionalInfoString .= ' pattern="' . $inputEntry['pattern'] . '"'; - - if(isset($inputEntry['title'])) - $additionalInfoString .= ' title="' . $inputEntry['title'] . '"'; - - if(!isset($inputEntry['exampleValue'])) - $inputEntry['exampleValue'] = ''; - - if(!isset($inputEntry['defaultValue'])) - $inputEntry['defaultValue'] = ''; - - $idArg = 'arg-' - . urlencode($bridgeName) - . '-' - . urlencode($parameterName) - . '-' - . urlencode($id); - - $card .= '<label for="' - . $idArg - . '">' - . $inputEntry['name'] - . ' : </label>' - . PHP_EOL; - - if(!isset($inputEntry['type']) || $inputEntry['type'] == 'text') { - $card .= '<input ' - . $additionalInfoString - . ' id="' - . $idArg - . '" type="text" value="' - . $inputEntry['defaultValue'] - . '" placeholder="' - . $inputEntry['exampleValue'] - . '" name="' - . $id - . '" /><br />' - . PHP_EOL; - } elseif($inputEntry['type'] == 'number') { - $card .= '<input ' - . $additionalInfoString - . ' id="' - . $idArg - . '" type="number" value="' - . $inputEntry['defaultValue'] - . '" placeholder="' - . $inputEntry['exampleValue'] - . '" name="' - . $id - . '" /><br />' - . PHP_EOL; - } else if($inputEntry['type'] == 'list') { - $card .= '<select ' - . $additionalInfoString - . ' id="' - . $idArg - . '" name="' - . $id - . '" >'; - - foreach($inputEntry['values'] as $name => $value) { - if(is_array($value)) { - $card .= '<optgroup label="' . htmlentities($name) . '">'; - foreach($value as $subname => $subvalue) { - if($inputEntry['defaultValue'] === $subname - || $inputEntry['defaultValue'] === $subvalue) { - $card .= '<option value="' - . $subvalue - . '" selected>' - . $subname - . '</option>'; - } else { - $card .= '<option value="' - . $subvalue - . '">' - . $subname - . '</option>'; - } - } - $card .= '</optgroup>'; - } else { - if($inputEntry['defaultValue'] === $name - || $inputEntry['defaultValue'] === $value) { - $card .= '<option value="' - . $value - . '" selected>' - . $name - . '</option>'; - } else { - $card .= '<option value="' - . $value - . '">' - . $name - . '</option>'; - } - } - } - $card .= '</select><br >'; - } elseif($inputEntry['type'] == 'checkbox') { - if($inputEntry['defaultValue'] === 'checked') - $card .= '<input ' - . $additionalInfoString - . ' id="' - . $idArg - . '" type="checkbox" name="' - . $id - . '" checked /><br />' - . PHP_EOL; - else - $card .= '<input ' - . $additionalInfoString - . ' id="' - . $idArg - . '" type="checkbox" name="' - . $id - . '" /><br />' - . PHP_EOL; - } - } - - if($isActive) { - if(defined('PROXY_URL') && PROXY_BYBRIDGE) { - $idArg = 'arg-' - . urlencode($bridgeName) - . '-' - . urlencode('proxyoff') - . '-' - . urlencode('_noproxy'); - - $card .= '<input id="' - . $idArg - . '" type="checkbox" name="_noproxy" />' - . PHP_EOL; - - $card .= '<label for="' - . $idArg - . '">Disable proxy (' - . ((defined('PROXY_NAME') && PROXY_NAME) ? PROXY_NAME : PROXY_URL) - . ')</label><br />' - . PHP_EOL; - } if(CUSTOM_CACHE_TIMEOUT) { - $idArg = 'arg-' - . urlencode($bridgeName) - . '-' - . urlencode('_cache_timeout'); - - $card .= '<label for="' - . $idArg - . '">Cache timeout in seconds : </label>' - . PHP_EOL; - - $card .= '<input id="' - . $idArg - . '" type="number" value="' - . $bridge->getCacheTimeout() - . '" name="_cache_timeout" /><br />' - . PHP_EOL; - } - $card .= $getHelperButtonsFormat($formats); - } else { - $card .= '<span style="font-weight: bold;">Inactive</span>'; - } - $card .= '</form>' . PHP_EOL; - } - - $card .= '<label class="showless" for="showmore-' . $bridgeName . '">Show less</label>'; - $card .= '<p class="maintainer">' . $bridge->getMaintainer() . '</p>'; - $card .= '</section>'; - - return $card; -} - function sanitize($textToSanitize, $removedTags = array('script', 'iframe', 'input', 'form'), $keptAttributes = array('title', 'href', 'src'), @@ -340,21 +40,137 @@ function backgroundToImg($htmlContent) { } +/** + * Convert relative links in HTML into absolute links + * @param $content HTML content to fix. Supports HTML objects or string objects + * @param $server full URL to the page containing relative links + * @return content with fixed URLs, as HTML object or string depending on input type + */ function defaultLinkTo($content, $server){ + $string_convert = false; + if (is_string($content)) { + $string_convert = true; + $content = str_get_html($content); + } + foreach($content->find('img') as $image) { - if(strpos($image->src, 'http') === false - && strpos($image->src, '//') === false - && strpos($image->src, 'data:') === false) - $image->src = $server . $image->src; + $image->src = urljoin($server, $image->src); } foreach($content->find('a') as $anchor) { - if(strpos($anchor->href, 'http') === false - && strpos($anchor->href, '//') === false - && strpos($anchor->href, '#') !== 0 - && strpos($anchor->href, '?') !== 0) - $anchor->href = $server . $anchor->href; + $anchor->href = urljoin($server, $anchor->href); + } + + if ($string_convert) { + $content = $content->outertext; } return $content; } + +/** + * Extract the first part of a string matching the specified start and end delimiters + * @param $string input string, e.g. '<div>Post author: John Doe</div>' + * @param $start start delimiter, e.g. 'author: ' + * @param $end end delimiter, e.g. '<' + * @return extracted string, e.g. 'John Doe', or false if the delimiters were not found. + */ +function extractFromDelimiters($string, $start, $end) { + if (strpos($string, $start) !== false) { + $section_retrieved = substr($string, strpos($string, $start) + strlen($start)); + $section_retrieved = substr($section_retrieved, 0, strpos($section_retrieved, $end)); + return $section_retrieved; + } return false; +} + +/** + * Remove one or more part(s) of a string using a start and end delmiters + * @param $string input string, e.g. 'foo<script>superscript()</script>bar' + * @param $start start delimiter, e.g. '<script' + * @param $end end delimiter, e.g. '</script>' + * @return cleaned string, e.g. 'foobar' + */ +function stripWithDelimiters($string, $start, $end) { + while(strpos($string, $start) !== false) { + $section_to_remove = substr($string, strpos($string, $start)); + $section_to_remove = substr($section_to_remove, 0, strpos($section_to_remove, $end) + strlen($end)); + $string = str_replace($section_to_remove, '', $string); + } + return $string; +} + +/** + * Remove HTML sections containing one or more sections using the same HTML tag + * @param $string input string, e.g. 'foo<div class="ads"><div>ads</div>ads</div>bar' + * @param $tag_name name of the HTML tag, e.g. 'div' + * @param $tag_start start of the HTML tag to remove, e.g. '<div class="ads">' + * @return cleaned string, e.g. 'foobar' + */ +function stripRecursiveHTMLSection($string, $tag_name, $tag_start){ + $open_tag = '<' . $tag_name; + $close_tag = '</' . $tag_name . '>'; + $close_tag_length = strlen($close_tag); + if(strpos($tag_start, $open_tag) === 0) { + while(strpos($string, $tag_start) !== false) { + $max_recursion = 100; + $section_to_remove = null; + $section_start = strpos($string, $tag_start); + $search_offset = $section_start; + do { + $max_recursion--; + $section_end = strpos($string, $close_tag, $search_offset); + $search_offset = $section_end + $close_tag_length; + $section_to_remove = substr($string, $section_start, $section_end - $section_start + $close_tag_length); + $open_tag_count = substr_count($section_to_remove, $open_tag); + $close_tag_count = substr_count($section_to_remove, $close_tag); + } while ($open_tag_count > $close_tag_count && $max_recursion > 0); + $string = str_replace($section_to_remove, '', $string); + } + } + return $string; +} + +/** + * Convert Markdown tags into HTML tags. Only a subset of the Markdown syntax is implemented. + * @param $string input string in Markdown format + * @return output string in HTML format + */ +function markdownToHtml($string) { + + //For more details about how these regex work: + // https://github.com/RSS-Bridge/rss-bridge/pull/802#discussion_r216138702 + // Images: https://regex101.com/r/JW9Evr/1 + // Links: https://regex101.com/r/eRGVe7/1 + // Bold: https://regex101.com/r/2p40Y0/1 + // Italic: https://regex101.com/r/xJkET9/1 + // Separator: https://regex101.com/r/ZBEqFP/1 + // Plain URL: https://regex101.com/r/2JHYwb/1 + // Site name: https://regex101.com/r/qIuKYE/1 + + $string = preg_replace('/\!\[([^\]]+)\]\(([^\) ]+)(?: [^\)]+)?\)/', '<img src="$2" alt="$1" />', $string); + $string = preg_replace('/\[([^\]]+)\]\(([^\)]+)\)/', '<a href="$2">$1</a>', $string); + $string = preg_replace('/\*\*(.*)\*\*/U', '<b>$1</b>', $string); + $string = preg_replace('/\*(.*)\*/U', '<i>$1</i>', $string); + $string = preg_replace('/__(.*)__/U', '<b>$1</b>', $string); + $string = preg_replace('/_(.*)_/U', '<i>$1</i>', $string); + $string = preg_replace('/[-]{6,99}/', '<hr />', $string); + $string = str_replace(' ', '<br />', $string); + $string = preg_replace('/([^"])(https?:\/\/[^ "<]+)([^"])/', '$1<a href="$2">$2</a>$3', $string . ' '); + $string = preg_replace('/([^"\/])(www\.[^ "<]+)([^"])/', '$1<a href="http://$2">$2</a>$3', $string . ' '); + + //As the regex are not perfect, we need to fix <i> and </i> that are introduced in URLs + // Fixup regex <i>: https://regex101.com/r/NTRPf6/1 + // Fixup regex </i>: https://regex101.com/r/aNklRp/1 + + $count = 1; + while($count > 0) { + $string = preg_replace('/ (src|href)="([^"]+)<i>([^"]+)"/U', ' $1="$2_$3"', $string, -1, $count); + } + + $count = 1; + while($count > 0) { + $string = preg_replace('/ (src|href)="([^"]+)<\/i>([^"]+)"/U', ' $1="$2_$3"', $string, -1, $count); + } + + return '<div>' . trim($string) . '</div>'; +} diff --git a/lib/validation.php b/lib/validation.php deleted file mode 100644 index fdcb51c..0000000 --- a/lib/validation.php +++ /dev/null @@ -1,95 +0,0 @@ -<?php -function validateData(&$data, $parameters){ - $validateTextValue = function($value, $pattern = null){ - if(!is_null($pattern)) { - $filteredValue = filter_var($value, - FILTER_VALIDATE_REGEXP, - array('options' => array( - 'regexp' => '/^' . $pattern . '$/' - ) - )); - } else { - $filteredValue = filter_var($value); - } - - if($filteredValue === false) - return null; - - return $filteredValue; - }; - - $validateNumberValue = function($value){ - $filteredValue = filter_var($value, FILTER_VALIDATE_INT); - - if($filteredValue === false) - return null; - - return $filteredValue; - }; - - $validateCheckboxValue = function($value){ - return filter_var($value, FILTER_VALIDATE_BOOLEAN, FILTER_NULL_ON_FAILURE); - }; - - $validateListValue = function($value, $expectedValues){ - $filteredValue = filter_var($value); - - if($filteredValue === false) - return null; - - if(!in_array($filteredValue, $expectedValues)) { // Check sub-values? - foreach($expectedValues as $subName => $subValue) { - if(is_array($subValue) && in_array($filteredValue, $subValue)) - return $filteredValue; - } - return null; - } - - return $filteredValue; - }; - - if(!is_array($data)) - return false; - - foreach($data as $name => $value) { - $registered = false; - foreach($parameters as $context => $set) { - if(array_key_exists($name, $set)) { - $registered = true; - if(!isset($set[$name]['type'])) { - $set[$name]['type'] = 'text'; - } - - switch($set[$name]['type']) { - case 'number': - $data[$name] = $validateNumberValue($value); - break; - case 'checkbox': - $data[$name] = $validateCheckboxValue($value); - break; - case 'list': - $data[$name] = $validateListValue($value, $set[$name]['values']); - break; - default: - case 'text': - if(isset($set[$name]['pattern'])) { - $data[$name] = $validateTextValue($value, $set[$name]['pattern']); - } else { - $data[$name] = $validateTextValue($value); - } - break; - } - - if(is_null($data[$name]) && isset($set[$name]['required']) && $set[$name]['required']) { - echo 'Parameter \'' . $name . '\' is invalid!' . PHP_EOL; - return false; - } - } - } - - if(!$registered) - return false; - } - - return true; -} diff --git a/phpcompatibility.xml b/phpcompatibility.xml new file mode 100644 index 0000000..f232523 --- /dev/null +++ b/phpcompatibility.xml @@ -0,0 +1,47 @@ +<?xml version="1.0" encoding="UTF-8"?> +<ruleset name="RSS-Bridge PHPCompatibility"> + <description>Defines rules for PHPCompatibility</description> + <exclude-pattern>./static</exclude-pattern> + <exclude-pattern>./vendor</exclude-pattern> + + <!-- Run against the PHPCompatibility ruleset --> + <!-- + + --> + <config name="testVersion" value="5.6"/> + <rule ref="PHPCompatibility"> + <!-- + "PHP 7 changes how most errors are reported by PHP. Instead of reporting + errors through the traditional error reporting mechanism used by PHP 5, + most errors are now reported by throwing Error exceptions." + + from: http://php.net/manual/en/language.errors.php7.php + + Skip this check for PHP 5.6 in order to support PHP 7.x + + Catch Exception and Error separately to support both versions. + + Example: + + <code> + try { + // Run your code here + } catch(Error $e) { + // Handle errors (PHP 7.0+) + } catch(Exception $e) { + // Handle exceptions (PHP 5.6+) + } + </code> + --> + <exclude name="PHPCompatibility.Classes.NewClasses.errorFound"/> + <!-- + RSS-Bridge uses parse_ini_file with INI_SCANNER_TYPED to load configuration + settings. INI_SCANNER_TYPED was added in PHP 5.6.1. Skip checking for that + specific constant. + + References: http://php.net/manual/de/function.parse-ini-file.php + --> + <exclude name="PHPCompatibility.Constants.NewConstants.ini_scanner_typedFound"/> + </rule> + +</ruleset> @@ -13,6 +13,13 @@ <rule ref="Generic.CodeAnalysis.UnnecessaryFinalModifier"/> <!-- Do not override methods to call their parent --> <rule ref="Generic.CodeAnalysis.UselessOverridingMethod"/> + <!-- Make sure the concatenation operator has spaces around it --> + <rule ref="Squiz.Strings.ConcatenationSpacing"> + <properties> + <property name="spacing" value="1"/> + <property name="ignoreNewlines" value="true"/> + </properties> + </rule> <!-- One line should not have more than 80 characters --> <!-- One line must never exceed 120 characters --> <rule ref="Generic.Files.LineLength"> diff --git a/phpunit.xml b/phpunit.xml new file mode 100644 index 0000000..16a082d --- /dev/null +++ b/phpunit.xml @@ -0,0 +1,16 @@ +<phpunit + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:noNamespaceSchemaLocation="http://schema.phpunit.de/4.5/phpunit.xsd" + colors="true" + processIsolation="false" + timeoutForSmallTests="1" + timeoutForMediumTests="1" + timeoutForLargeTests="6" > + + <testsuites> + <testsuite name="Standard test suite"> + <file>tests/BridgeImplementationTest.php</file> + </testsuite> + </testsuites> + +</phpunit> diff --git a/static/HtmlFormat.css b/static/HtmlFormat.css index 195a9b0..e17e325 100644 --- a/static/HtmlFormat.css +++ b/static/HtmlFormat.css @@ -1,119 +1,87 @@ html, body, div, span, applet, object, iframe, h1, h2, h3, h4, h5, h6, p, blockquote, pre, a, abbr, acronym, address, big, cite, code, del, dfn, em, img, ins, kbd, q, s, samp, small, strike, strong, sub, sup, tt, var, b, u, i, center, dl, dt, dd, ol, ul, li, fieldset, form, label, legend, table, caption, tbody, tfoot, thead, tr, th, td, article, aside, canvas, details, figcaption, figure, footer, header, hgroup, menu, nav, section, summary, time, mark, audio, video { - - margin: 0; - padding: 0; - border: 0; - outline: 0; - font-size: 100%; - vertical-align: baseline; - + margin: 0; + padding: 0; + border: 0; + outline: 0; + font-size: 100%; + font: inherit; + vertical-align: baseline; } /* HTML5 display-role reset for older browsers */ -article, aside, details, figcaption, figure, footer, header, hgroup, menu, nav, section { - - display: block; - + article, aside, details, figcaption, figure, footer, header, hgroup, menu, nav, section { + display: block; } - /* Let's go for the actual style */ - -body { - - background-color: #EEEEEE; - font-family: 'Noto Sans'; - -} - -section { - + body { + background-color: #f0f0f0; + font-family: -apple-system,BlinkMacSystemFont,"Segoe UI",Roboto,Helvetica,Arial,sans-serif,"Apple Color Emoji","Segoe UI Emoji","Segoe UI Symbol"; +} + a, a:link, a:visited { + color: #2196F3; + text-decoration: none; +} + a:hover { + text-decoration: underline; +} + img { + max-width: 100%; +} +/* Section */ + section { background-color: #FFFFFF; - width: 90%; + width: 60%; margin: 30px auto; - padding: 10px 15px; - - box-shadow: 0px 1px 2px rgba(0,0,0, 0.25); - -} - -section > h2 { - + padding: 15px 15px; + text-align: center; + box-shadow: 0 6px 15px rgba(0, 0, 0, 0.09); + border-radius: 4px; +} + section > h2 { font-size: 200%; font-weight: bold; text-align: center; - } - -h1.pagetitle { - + h1.pagetitle { + margin: 40px 0 20px; font-size: 300%; font-weight: bold; - text-align: center; color: #2196F3; - } - -h1.pagetitle > a { + h1.pagetitle > a { color: #2196F3; } - -a.backlink, a.backlink:link, a.backlink:visited, a.itemtitle, a.itemtitle:link, a.itemtitle:visited { - + a.backlink, a.backlink:link, a.backlink:visited, a.itemtitle, a.itemtitle:link, a.itemtitle:visited { color: #2196F3; - } - -.buttons { - + .buttons { text-align: center; - } - -section > div.content, section > div.categories, -section > div.content, section > div.attachments { - + section > div.content, section > div.attachments { padding: 10px; - } - -section > div.categories > li.category, -section > div.attachments > li.enclosure { - + section > div.attachments > li.enclosure { list-style-type: circle; list-style-position: inside; - } - -section > time, section > p.author { - + section > time, section > p.author { color: #888; font-size: 80%; padding: 10px; - } - -button.backbutton, button.rss-feed { - - line-height: 1em; + button { + line-height: 1.9em; color: #FFF; font-weight: bold; vertical-align: middle; padding: 6px 12px; margin: 12px auto 0px; - box-shadow: 0px 1px 2px rgba(0, 0, 0, 0.3); - border-radius: 2px; + border-radius: 4px; border: 1px solid transparent; - width: 200px; background: #2196F3 none repeat scroll 0% 0%; cursor: pointer; - - margin: 10px; - - -} - -img { - - max-width: 100%; - + width: 200px; } + button:hover { + background: #49afff; +}
\ No newline at end of file diff --git a/static/search.js b/static/search.js index 3ddd99b..daf3287 100644 --- a/static/search.js +++ b/static/search.js @@ -3,20 +3,53 @@ function search() { var searchTerm = document.getElementById('searchfield').value; var searchableElements = document.getElementsByTagName('section'); - var regexMatch = new RegExp(searchTerm, "i"); + var regexMatch = new RegExp(searchTerm, 'i'); + + // Attempt to create anchor from search term (will default to 'localhost' on failure) + var searchTermUri = document.createElement('a'); + searchTermUri.href = searchTerm; + + if(searchTermUri.hostname == 'localhost') { + searchTermUri = null; + } else { + + // Ignore "www." + if(searchTermUri.hostname.indexOf('www.') === 0) { + searchTermUri.hostname = searchTermUri.hostname.substr(4); + } + + } for(var i = 0; i < searchableElements.length; i++) { var textValue = searchableElements[i].getAttribute('data-ref'); - if(textValue != null) { + var anchors = searchableElements[i].getElementsByTagName('a'); + + if(anchors != null && anchors.length > 0) { + + var uriValue = anchors[0]; // First anchor is bridge URI + + // Ignore "www." + if(uriValue.hostname.indexOf('www.') === 0) { + uriValue.hostname = uriValue.hostname.substr(4); + } + + } + + if(textValue != null || uriValue != null) { - if(textValue.match(regexMatch) == null && searchableElements[i].style.display != "none") { + if(textValue.match(regexMatch) != null || + uriValue.hostname.match(regexMatch) || + searchTermUri != null && + uriValue.hostname != 'localhost' && ( + uriValue.href.match(regexMatch) != null || + uriValue.hostname == searchTermUri.hostname)) { - searchableElements[i].style.display = "none"; + searchableElements[i].style.display = 'block'; - } else if(textValue.match(regexMatch) != null) { + } else { - searchableElements[i].style.display = "block"; + searchableElements[i].style.display = 'none'; } diff --git a/static/style.css b/static/style.css index ac2f469..fa70f2d 100644 --- a/static/style.css +++ b/static/style.css @@ -1,310 +1,306 @@ html, body, div, span, applet, object, iframe, h1, h2, h3, h4, h5, h6, p, blockquote, pre, a, abbr, acronym, address, big, cite, code, del, dfn, em, img, ins, kbd, q, s, samp, small, strike, strong, sub, sup, tt, var, b, u, i, center, dl, dt, dd, ol, ul, li, fieldset, form, label, legend, table, caption, tbody, tfoot, thead, tr, th, td, article, aside, canvas, details, figcaption, figure, footer, header, hgroup, menu, nav, section, summary, time, mark, audio, video { - - margin: 0; - padding: 0; - border: 0; - outline: 0; - font-size: 100%; - font: inherit; - vertical-align: baseline; - + margin: 0; + padding: 0; + border: 0; + outline: 0; + font-size: 100%; + font: inherit; + vertical-align: baseline; } + /* HTML5 display-role reset for older browsers */ article, aside, details, figcaption, figure, footer, header, hgroup, menu, nav, section { + display: block; +} - display: block; +/* Adjust parameters for browsers that don't support the grid layout */ +.parameters label:before { + content: " "; + display: block; } /* Let's go for the actual style */ - body { + background-color: #f0f0f0; + font-family: -apple-system,BlinkMacSystemFont,"Segoe UI",Roboto,Helvetica,Arial,sans-serif,"Apple Color Emoji","Segoe UI Emoji","Segoe UI Symbol"; +} - background-color: #EEEEEE; - font-family: 'Noto Sans'; +a, a:link, a:visited { + color: #2196F3; + text-decoration: none; +} +a:hover { + text-decoration: underline; } -header { +/* Header */ - text-shadow:0 5px 6px rgba(150,150,150,0.69); +header { + margin-top: 40px; text-align: center; color: #1182DB; - } header > h1 { - - font-size: 300%; - + font-size: 500%; + font-weight: bold; } header > h2 { - margin-left: 1em; - font-size: 120%; + font-size: 200%; +} +header > section.warning { + width: 40%; + background-color: #ffc600; + color: #5f5f5f; } -header > p.status { +header > section.critical-warning { + width: 40%; + background-color: #cf3e3e; font-weight: bold; - margin: 1em; - color: red; + color: white; } -input[type="text"] { - +select, +input[type="text"], +input[type="number"] { background-color: white; color: #404552; - border: 0px; - border-bottom: 2px solid #2196F3; - font-size: 1.1em; + border: 1px solid #dedede; margin-left: 8px; - padding-left: 4px; + margin-bottom: 10px; + padding: 5px 10px; +} +select:focus, +input[type="text"]:focus, +input[type="number"]:focus { + outline: none; + border-color: #888; } .searchbar { - - width: 50%; - margin: auto; - + width: 40%; + margin: 40px auto 100px; } .searchbar input[type="text"] { - - width: 100%; + width: 90%; margin: auto; - font-size: 1.4em; + font-size: 1.1em; text-align: center; - + margin-bottom: 10px; } .searchbar input[type="text"]::placeholder { - text-align: center; - -} - -.searchbar input[type="text"]:focus::-webkit-input-placeholder { - - opacity: 0; - -} - -.searchbar input[type="text"]:focus::-moz-placeholder { - - opacity: 0; - -} - -.searchbar input[type="text"]:focus:-moz-placeholder { - - opacity: 0; - } +.searchbar input[type="text"]:focus::-webkit-input-placeholder, +.searchbar input[type="text"]:focus::-moz-placeholder, +.searchbar input[type="text"]:focus:-moz-placeholder, .searchbar input[type="text"]:focus:-ms-input-placeholder { - opacity: 0; - } .searchbar > h3 { - - font-size: 150%; + font-size: 200%; font-weight: bold; color: #1182DB; - + margin-bottom: 10px; } +/* Section */ section { - background-color: #FFFFFF; - width: 80%; + width: 60%; margin: 30px auto; - padding: 10px 15px; + padding: 15px 15px; text-align: center; - - box-shadow: 0px 1px 2px rgba(0,0,0, 0.25); - + box-shadow: 0 6px 15px rgba(0, 0, 0, 0.09); + border-radius: 4px; } - section.footer { - opacity: 0.5; - } section.footer:hover { - opacity: 1; - } section.footer .version { - font-size: 80%; - } section > h2 { - font-size: 200%; font-weight: bold; - -} - -a, a:link, a:visited { - - color: #2196F3; - } +/* Buttons */ button { - line-height: 1.9em; color: #FFF; font-weight: bold; vertical-align: middle; padding: 6px 12px; margin: 12px auto 0px; - box-shadow: 0px 1px 2px rgba(0, 0, 0, 0.3); - border-radius: 2px; + border-radius: 4px; border: 1px solid transparent; - min-width: 140px; background: #2196F3 none repeat scroll 0% 0%; cursor: pointer; - width: calc(20% - 4px); - } button.small { - width: auto; line-height: 1.2em; +} +button:hover { + background: #49afff; } .description { - margin: 10px; - text-decoration: underline; - } h5 { - margin: 20px; font-weight: bold; - } form { - margin-bottom: 6px; +} +.parameters label::first-letter { + text-transform: capitalize; } -.maintainer { +.parameters label::after { + content: ' :'; +} - font-size: 60%; - text-align: right; +@supports (display: grid) { + + .parameters { + display: grid; + padding: 12px 0; + grid-template-columns: 40% max-content; + grid-column-gap: 10px; + grid-row-gap: 5px; + } + + .parameters label { + text-align: right; + } + + .parameters label::before { + content: none; + } + .parameters input[type="text"], + .parameters input[type="number"], + .parameters input[type="checkbox"], + .parameters select { + margin-left: 0; + } + + .parameters input[type="text"], + .parameters input[type="number"] { + width: auto; + color: #404552; + } + + .parameters input[type="checkbox"] { + width: 20px; + height: 20px; + } + +} /* @supports (display: grid) */ + +.maintainer { + color: #888888; + font-size: 70%; + text-align: right; } .secure-warning { - background-color: #ffc600; color: #5f5f5f; - box-shadow: 0px 1px 2px rgba(0, 0, 0, 0.3); border-radius: 2px; border: 1px solid transparent; - width: 80%; margin: auto; margin-bottom: 6px; - } form { - display: none; +} +select { + padding: 5px 10px; + margin-left: 8px; } h5 { - display: none; - } +/* Show more / less */ .showmore-box { - display: none; - } .showmore, .showless { - color: #888888; cursor: pointer; - +} +.showmore:hover, .showless:hover { + color: #000; + cursor: pointer; } .showmore-box:checked ~ .showmore { - display: none; - } .showmore-box:not(:checked) ~ .showless { - display: none; - } - - .showmore-box:checked ~ form, .showmore-box:checked ~ h5 { - display: block; - } /* Additional styles for error pages */ - .exception-message { - background-color: #c00000; color: #FFFFFF; - font-weight: bold; - box-shadow: 0px 1px 2px rgba(0, 0, 0, 0.3); border-radius: 2px; border: 1px solid transparent; - width: 80%; margin: auto; margin-bottom: 6px; - } .advice { - margin-left: auto; margin-right: auto; - display: table; - } .advice > li { - text-align: left; - } diff --git a/tests/BridgeImplementationTest.php b/tests/BridgeImplementationTest.php new file mode 100644 index 0000000..b7d13f1 --- /dev/null +++ b/tests/BridgeImplementationTest.php @@ -0,0 +1,191 @@ +<?php + +use PHPUnit\Framework\TestCase; +use PHPUnit\Framework\TestResult; +use PHPUnit\Framework\AssertionFailedError; + +require_once(__DIR__ . '/../lib/RssBridge.php'); + +Bridge::setDir(PATH_LIB_BRIDGES); + +/** + * This class checks bridges for implementation details: + * + * - A bridge must not implement public functions other than the ones specified + * by the bridge interfaces. Custom functions must be defined in private or + * protected scope. + * - getName() must return a valid string (non-empty) + * - getURI() must return a valid URI + * - A bridge must define constants for NAME, URI, DESCRIPTION and MAINTAINER, + * CACHE_TIMEOUT and PARAMETERS are optional + */ +final class BridgeImplementationTest extends TestCase { + + private function CheckBridgePublicFunctions($bridgeName){ + + $parent_methods = array(); + + if(in_array('BridgeInterface', class_parents($bridgeName))) { + $parent_methods = array_merge($parent_methods, get_class_methods('BridgeInterface')); + } + + if(in_array('BridgeAbstract', class_parents($bridgeName))) { + $parent_methods = array_merge($parent_methods, get_class_methods('BridgeAbstract')); + } + + if(in_array('FeedExpander', class_parents($bridgeName))) { + $parent_methods = array_merge($parent_methods, get_class_methods('FeedExpander')); + } + + // Receive all non abstract methods + $methods = array_diff(get_class_methods($bridgeName), $parent_methods); + $method_names = implode(', ', $methods); + + $errmsg = $bridgeName + . ' implements additional public method(s): ' + . $method_names + . '! Custom functions must be defined in private or protected scope!'; + + $this->assertEmpty($method_names, $errmsg); + + } + + private function CheckBridgeGetNameDefaultValue($bridgeName){ + + if(in_array('BridgeAbstract', class_parents($bridgeName))) { // Is bridge + + if(!$this->isFunctionMemberOf($bridgeName, 'getName')) + return; + + $bridge = new $bridgeName(); + $abstract = new BridgeAbstractTest(); + + $message = $bridgeName . ': \'getName\' must return a valid name!'; + + $this->assertNotEmpty(trim($bridge->getName()), $message); + + } + + } + + // Checks whether the getURI function returns empty or default values + private function CheckBridgeGetURIDefaultValue($bridgeName){ + + if(in_array('BridgeAbstract', class_parents($bridgeName))) { // Is bridge + + if(!$this->isFunctionMemberOf($bridgeName, 'getURI')) + return; + + $bridge = new $bridgeName(); + $abstract = new BridgeAbstractTest(); + + $message = $bridgeName . ': \'getURI\' must return a valid URI!'; + + $this->assertNotEmpty(trim($bridge->getURI()), $message); + + } + + } + + private function CheckBridgePublicConstants($bridgeName){ + + // Assertion only works for BridgeAbstract! + if(in_array('BridgeAbstract', class_parents($bridgeName))) { + + $ref = new ReflectionClass($bridgeName); + $constants = $ref->getConstants(); + + $ref = new ReflectionClass('BridgeAbstract'); + $parent_constants = $ref->getConstants(); + + foreach($parent_constants as $key => $value) { + + $this->assertArrayHasKey($key, $constants, 'Constant ' . $key . ' missing in ' . $bridgeName); + + // Skip optional constants + if($key !== 'PARAMETERS' && $key !== 'CACHE_TIMEOUT') { + $this->assertNotEquals($value, $constants[$key], 'Constant ' . $key . ' missing in ' . $bridgeName); + } + + } + + } + + } + + private function isFunctionMemberOf($bridgeName, $functionName){ + + $bridgeReflector = new ReflectionClass($bridgeName); + $bridgeMethods = $bridgeReflector->GetMethods(); + $bridgeHasMethod = false; + + foreach($bridgeMethods as $method) { + + if($method->name === $functionName && $method->class === $bridgeReflector->name) { + return true; + } + + } + + return false; + + } + + public function testBridgeImplementation($bridgeName){ + + require_once('bridges/' . $bridgeName . '.php'); + + $this->CheckBridgePublicFunctions($bridgeName); + $this->CheckBridgePublicConstants($bridgeName); + $this->CheckBridgeGetNameDefaultValue($bridgeName); + $this->CheckBridgeGetURIDefaultValue($bridgeName); + + } + + public function count() { + return count(Bridge::listBridges()); + } + + public function run(TestResult $result = null) { + + if ($result === null) { + $result = new TestResult; + } + + foreach (Bridge::listBridges() as $bridge) { + + $bridge .= 'Bridge'; + + $result->startTest($this); + PHP_Timer::start(); + $stopTime = null; + + try { + $this->testBridgeImplementation($bridge); + } catch (AssertionFailedError $e) { + + $stopTime = PHP_Timer::stop(); + $result->addFailure($this, $e, $stopTime); + + } catch (Exception $e) { + + $stopTime = PHP_Timer::stop(); + $result->addError($this, $e, $stopTime); + + } + + if ($stopTime === null) { + $stopTime = PHP_Timer::stop(); + } + + $result->endTest($this, $stopTime); + + } + + return $result; + } +} + +class BridgeAbstractTest extends BridgeAbstract { + public function collectData(){} +} diff --git a/vendor/php-urljoin/src/urljoin.php b/vendor/php-urljoin/src/urljoin.php new file mode 100644 index 0000000..ef84fbc --- /dev/null +++ b/vendor/php-urljoin/src/urljoin.php @@ -0,0 +1,140 @@ +<?php + +/* + +A spiritual port of Python's urlparse.urljoin() function to PHP. Why this isn't in the standard library is anyone's guess. + +Author: fluffy, http://beesbuzz.biz/ +Latest version at: https://github.com/plaidfluff/php-urljoin + + */ + +function urljoin($base, $rel) { + if (!$base) { + return $rel; + } + + if (!$rel) { + return $base; + } + + $uses_relative = array('', 'ftp', 'http', 'gopher', 'nntp', 'imap', + 'wais', 'file', 'https', 'shttp', 'mms', + 'prospero', 'rtsp', 'rtspu', 'sftp', + 'svn', 'svn+ssh', 'ws', 'wss'); + + $pbase = parse_url($base); + $prel = parse_url($rel); + + if ($prel === false || preg_match('/^[a-z0-9\-.]*[^a-z0-9\-.:][a-z0-9\-.]*:/i', $rel)) { + /* + Either parse_url couldn't parse this, or the original URL + fragment had an invalid scheme character before the first :, + which can confuse parse_url + */ + $prel = array('path' => $rel); + } + + if (array_key_exists('path', $pbase) && $pbase['path'] === '/') { + unset($pbase['path']); + } + + if (isset($prel['scheme'])) { + if ($prel['scheme'] != $pbase['scheme'] || in_array($prel['scheme'], $uses_relative) == false) { + return $rel; + } + } + + $merged = array_merge($pbase, $prel); + + // Handle relative paths: + // 'path/to/file.ext' + // './path/to/file.ext' + if (array_key_exists('path', $prel) && substr($prel['path'], 0, 1) != '/') { + + // Normalize: './path/to/file.ext' => 'path/to/file.ext' + if (substr($prel['path'], 0, 2) === './') { + $prel['path'] = substr($prel['path'], 2); + } + + if (array_key_exists('path', $pbase)) { + $dir = preg_replace('@/[^/]*$@', '', $pbase['path']); + $merged['path'] = $dir . '/' . $prel['path']; + } else { + $merged['path'] = '/' . $prel['path']; + } + + } + + if(array_key_exists('path', $merged)) { + // Get the path components, and remove the initial empty one + $pathParts = explode('/', $merged['path']); + array_shift($pathParts); + + $path = []; + $prevPart = ''; + foreach ($pathParts as $part) { + if ($part == '..' && count($path) > 0) { + // Cancel out the parent directory (if there's a parent to cancel) + $parent = array_pop($path); + // But if it was also a parent directory, leave it in + if ($parent == '..') { + array_push($path, $parent); + array_push($path, $part); + } + } else if ($prevPart != '' || ($part != '.' && $part != '')) { + // Don't include empty or current-directory components + if ($part == '.') { + $part = ''; + } + array_push($path, $part); + } + $prevPart = $part; + } + $merged['path'] = '/' . implode('/', $path); + } + + $ret = ''; + if (isset($merged['scheme'])) { + $ret .= $merged['scheme'] . ':'; + } + + if (isset($merged['scheme']) || isset($merged['host'])) { + $ret .= '//'; + } + + if (isset($prel['host'])) { + $hostSource = $prel; + } else { + $hostSource = $pbase; + } + + // username, password, and port are associated with the hostname, not merged + if (isset($hostSource['host'])) { + if (isset($hostSource['user'])) { + $ret .= $hostSource['user']; + if (isset($hostSource['pass'])) { + $ret .= ':' . $hostSource['pass']; + } + $ret .= '@'; + } + $ret .= $hostSource['host']; + if (isset($hostSource['port'])) { + $ret .= ':' . $hostSource['port']; + } + } + + if (isset($merged['path'])) { + $ret .= $merged['path']; + } + + if (isset($prel['query'])) { + $ret .= '?' . $prel['query']; + } + + if (isset($prel['fragment'])) { + $ret .= '#' . $prel['fragment']; + } + + return $ret; +} |