From b7d5672d68b0d824354249534ea47fa64803751d Mon Sep 17 00:00:00 2001 From: Johannes Schauer Date: Fri, 4 Aug 2017 22:06:01 +0200 Subject: Import rss-bridge_2017-08-03-1.debian.tar.xz [dgit import tarball rss-bridge 2017-08-03-1 rss-bridge_2017-08-03-1.debian.tar.xz] --- changelog | 5 +++++ compat | 1 + control | 20 ++++++++++++++++++++ copyright | 52 ++++++++++++++++++++++++++++++++++++++++++++++++++++ rss-bridge.dirs | 1 + rss-bridge.docs | 1 + rss-bridge.install | 8 ++++++++ rss-bridge.links | 2 ++ rules | 11 +++++++++++ source/format | 1 + whitelist.txt | 16 ++++++++++++++++ 11 files changed, 118 insertions(+) create mode 100644 changelog create mode 100644 compat create mode 100644 control create mode 100644 copyright create mode 100644 rss-bridge.dirs create mode 100644 rss-bridge.docs create mode 100644 rss-bridge.install create mode 100644 rss-bridge.links create mode 100755 rules create mode 100644 source/format create mode 100644 whitelist.txt diff --git a/changelog b/changelog new file mode 100644 index 0000000..52d9b8a --- /dev/null +++ b/changelog @@ -0,0 +1,5 @@ +rss-bridge (2017-08-03-1) unstable; urgency=medium + + * Initial release. (Closes: #870687) + + -- Johannes Schauer Fri, 04 Aug 2017 22:06:01 +0200 diff --git a/compat b/compat new file mode 100644 index 0000000..f599e28 --- /dev/null +++ b/compat @@ -0,0 +1 @@ +10 diff --git a/control b/control new file mode 100644 index 0000000..30f3346 --- /dev/null +++ b/control @@ -0,0 +1,20 @@ +Source: rss-bridge +Section: web +Priority: extra +Maintainer: Johannes Schauer +Homepage: https://github.com/RSS-Bridge/rss-bridge +Vcs-Browser: https://browse.dgit.debian.org/rss-bridge.git/ +Vcs-Git: https://git.dgit.debian.org/rss-bridge +Standards-Version: 4.0.0 +Build-Depends: debhelper (>= 10) + +Package: rss-bridge +Architecture: all +Depends: ${misc:Depends}, php5-curl, php5-json, nginx | httpd +Description: web service generating ATOM feeds for websites that don't have them + Provides a PHP web service which generates ATOM feeds for facebook, twitter, + youtube, flickr, google, instagram, pinterest and more than 130 other web + services which do not provide ATOM or RSS feeds themselves. The software acts + as a proxy between the web service and an RSS reader software. It scrapes + their publicly available HTML pages to extract the necessary information. + To prevent banning, the results are cached. diff --git a/copyright b/copyright new file mode 100644 index 0000000..99c89e4 --- /dev/null +++ b/copyright @@ -0,0 +1,52 @@ +Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ +Upstream-Name: rss-bridge +Upstream-Contact: https://github.com/RSS-Bridge/rss-bridge/issues +Source: https://github.com/RSS-Bridge/rss-bridge + +Files: * +Copyright: 2013-2014 sebsauvage + 2014-2017 Mitsukarenai + 2016-2017 Pierre Mazière + 2015-2017 logmanoriginal +License: public-domain + This is free and unencumbered software released into the public domain. + . + Anyone is free to copy, modify, publish, use, compile, sell, or + distribute this software, either in source code form or as a compiled + binary, for any purpose, commercial or non-commercial, and by any + means. + . + In jurisdictions that recognize copyright laws, the author or authors + of this software dedicate any and all copyright interest in the + software to the public domain. We make this dedication for the benefit + of the public at large and to the detriment of our heirs and + successors. We intend this dedication to be an overt act of + relinquishment in perpetuity of all present and future rights to this + software under copyright law. + +Files: debian/* +Copyright: 2017 Johannes Schauer +License: Expat + +Files: vendor/simplehtmldom/simple_html_dom.php +Copyright: S.C. Chen + John Schlick + Rus Carroll + Jose Solorzano + Yousuke Kumakura + Vadim Voituk + Antcs +License: Expat +Comment: https://sourceforge.net/p/simplehtmldom/feature-requests/47/ + +License: Expat + Permission is hereby granted, free of charge, to any person obtaining + a copy of this software and associated documentation files (the + "Software"), to deal in the Software without restriction, including + without limitation the rights to use, copy, modify, merge, publish, + distribute, sublicense, and/or sell copies of the Software, and to + permit persons to whom the Software is furnished to do so, subject to + the following conditions: + . + The above copyright notice and this permission notice shall be included + in all copies or substantial portions of the Software. diff --git a/rss-bridge.dirs b/rss-bridge.dirs new file mode 100644 index 0000000..72c62be --- /dev/null +++ b/rss-bridge.dirs @@ -0,0 +1 @@ +/var/cache/rss-bridge diff --git a/rss-bridge.docs b/rss-bridge.docs new file mode 100644 index 0000000..b43bf86 --- /dev/null +++ b/rss-bridge.docs @@ -0,0 +1 @@ +README.md diff --git a/rss-bridge.install b/rss-bridge.install new file mode 100644 index 0000000..266ba42 --- /dev/null +++ b/rss-bridge.install @@ -0,0 +1,8 @@ +./bridges /usr/share/rss-bridge +./lib /usr/share/rss-bridge +./index.php /usr/share/rss-bridge +./formats /usr/share/rss-bridge +./caches /usr/share/rss-bridge +./static /usr/share/rss-bridge +./vendor /usr/share/rss-bridge +debian/whitelist.txt /etc/rss-bridge diff --git a/rss-bridge.links b/rss-bridge.links new file mode 100644 index 0000000..fce5361 --- /dev/null +++ b/rss-bridge.links @@ -0,0 +1,2 @@ +/var/cache/rss-bridge /usr/share/rss-bridge/cache +/etc/rss-bridge/whitelist.txt /usr/share/rss-bridge/whitelist.txt diff --git a/rules b/rules new file mode 100755 index 0000000..27263de --- /dev/null +++ b/rules @@ -0,0 +1,11 @@ +#!/usr/bin/make -f +%: + dh $@ + +override_dh_install: + dh_install + install -d -o www-data -g www-data $(CURDIR)/debian/rss-bridge/var/cache/rss-bridge + chmod -x $(CURDIR)/debian/rss-bridge/usr/share/rss-bridge/bridges/LeBonCoinBridge.php + +override_dh_fixperms: + dh_fixperms --exclude /var/cache/rss-bridge diff --git a/source/format b/source/format new file mode 100644 index 0000000..163aaf8 --- /dev/null +++ b/source/format @@ -0,0 +1 @@ +3.0 (quilt) diff --git a/whitelist.txt b/whitelist.txt new file mode 100644 index 0000000..ffb536f --- /dev/null +++ b/whitelist.txt @@ -0,0 +1,16 @@ +BandcampBridge +CryptomeBridge +DansTonChatBridge +DuckDuckGoBridge +FacebookBridge +FlickrExploreBridge +GooglePlusPostBridge +GoogleSearchBridge +IdenticaBridge +InstagramBridge +OpenClassroomsBridge +PinterestBridge +ScmbBridge +TwitterBridge +WikipediaBridge +YoutubeBridge -- cgit v1.2.3 From b005331cd910c0cc7dee2ddf82491b8248f431cf Mon Sep 17 00:00:00 2001 From: Johannes Schauer Date: Fri, 4 Aug 2017 22:06:01 +0200 Subject: Import rss-bridge_2017-08-03.orig.tar.gz [dgit import orig rss-bridge_2017-08-03.orig.tar.gz] --- .gitattributes | 22 + .gitignore | 234 ++++ .travis.yml | 19 + CHANGELOG.md | 233 ++++ CONTRIBUTING.md | 47 + README.md | 142 +++ UNLICENSE | 25 + bridges/ABCTabsBridge.php | 42 + bridges/AcrimedBridge.php | 25 + bridges/AllocineFRBridge.php | 87 ++ bridges/AmazonBridge.php | 94 ++ bridges/AnimeUltimeBridge.php | 135 +++ bridges/Arte7Bridge.php | 102 ++ bridges/AskfmBridge.php | 74 ++ bridges/BandcampBridge.php | 63 ++ bridges/BastaBridge.php | 34 + bridges/BlaguesDeMerdeBridge.php | 31 + bridges/BooruprojectBridge.php | 45 + bridges/CADBridge.php | 45 + bridges/CNETBridge.php | 93 ++ bridges/CastorusBridge.php | 118 ++ bridges/CollegeDeFranceBridge.php | 84 ++ bridges/CommonDreamsBridge.php | 26 + bridges/CopieDoubleBridge.php | 35 + bridges/CourrierInternationalBridge.php | 55 + bridges/CpasbienBridge.php | 74 ++ bridges/CryptomeBridge.php | 45 + bridges/DailymotionBridge.php | 123 +++ bridges/DanbooruBridge.php | 62 ++ bridges/DansTonChatBridge.php | 23 + bridges/DauphineLibereBridge.php | 56 + bridges/DemoBridge.php | 46 + bridges/DeveloppezDotComBridge.php | 47 + bridges/DiceBridge.php | 120 ++ bridges/DilbertBridge.php | 36 + bridges/DollbooruBridge.php | 9 + bridges/DuckDuckGoBridge.php | 42 + bridges/EZTVBridge.php | 67 ++ bridges/EliteDangerousGalnetBridge.php | 35 + bridges/ElsevierBridge.php | 75 ++ bridges/EstCeQuonMetEnProdBridge.php | 37 + bridges/EtsyBridge.php | 83 ++ bridges/FB2Bridge.php | 281 +++++ bridges/FacebookBridge.php | 305 ++++++ bridges/FeedExpanderExampleBridge.php | 62 ++ bridges/FierPandaBridge.php | 24 + bridges/FilterBridge.php | 77 ++ bridges/FlickrBridge.php | 120 ++ bridges/FootitoBridge.php | 75 ++ bridges/FourchanBridge.php | 78 ++ bridges/FuturaSciencesBridge.php | 173 +++ bridges/GBAtempBridge.php | 157 +++ bridges/GelbooruBridge.php | 22 + bridges/GiphyBridge.php | 76 ++ bridges/GithubIssueBridge.php | 192 ++++ bridges/GithubSearchBridge.php | 50 + bridges/GizmodoBridge.php | 36 + bridges/GoComicsBridge.php | 59 + bridges/GooglePlusPostBridge.php | 111 ++ bridges/GoogleSearchBridge.php | 64 ++ bridges/HDWallpapersBridge.php | 83 ++ bridges/HentaiHavenBridge.php | 37 + bridges/IdenticaBridge.php | 52 + bridges/InstagramBridge.php | 93 ++ bridges/IsoHuntBridge.php | 465 ++++++++ bridges/JapanExpoBridge.php | 100 ++ bridges/KATBridge.php | 123 +++ bridges/KernelBugTrackerBridge.php | 152 +++ bridges/KonachanBridge.php | 11 + bridges/KoreusBridge.php | 22 + bridges/KununuBridge.php | 249 +++++ bridges/LWNprevBridge.php | 265 +++++ bridges/LeBonCoinBridge.php | 190 ++++ bridges/LeMondeInformatiqueBridge.php | 44 + bridges/LegifranceJOBridge.php | 68 ++ bridges/LesJoiesDuCodeBridge.php | 45 + bridges/LichessBridge.php | 31 + bridges/LinkedInCompanyBridge.php | 37 + bridges/LolibooruBridge.php | 11 + bridges/MangareaderBridge.php | 249 +++++ bridges/MilbooruBridge.php | 11 + bridges/MixCloudBridge.php | 52 + bridges/MoebooruBridge.php | 56 + bridges/MoinMoinBridge.php | 327 ++++++ bridges/MondeDiploBridge.php | 26 + bridges/MsnMondeBridge.php | 35 + bridges/MspabooruBridge.php | 12 + bridges/NasaApodBridge.php | 44 + bridges/NeuviemeArtBridge.php | 57 + bridges/NextInpactBridge.php | 34 + bridges/NextgovBridge.php | 74 ++ bridges/NiceMatinBridge.php | 32 + bridges/NovelUpdatesBridge.php | 69 ++ bridges/OpenClassroomsBridge.php | 49 + bridges/ParuVenduImmoBridge.php | 102 ++ bridges/PickyWallpapersBridge.php | 101 ++ bridges/PinterestBridge.php | 163 +++ bridges/PlanetLibreBridge.php | 38 + bridges/RTBFBridge.php | 66 ++ bridges/RainbowSixSiegeBridge.php | 36 + bridges/ReadComicsBridge.php | 44 + bridges/Releases3DSBridge.php | 136 +++ bridges/ReporterreBridge.php | 47 + bridges/Rue89Bridge.php | 25 + bridges/Rule34Bridge.php | 12 + bridges/Rule34pahealBridge.php | 10 + bridges/SafebooruBridge.php | 12 + bridges/SakugabooruBridge.php | 11 + bridges/ScmbBridge.php | 39 + bridges/ScoopItBridge.php | 42 + bridges/SensCritiqueBridge.php | 97 ++ bridges/SexactuBridge.php | 88 ++ bridges/ShanaprojectBridge.php | 123 +++ bridges/Shimmie2Bridge.php | 39 + bridges/SoundcloudBridge.php | 64 ++ bridges/SteamBridge.php | 129 +++ bridges/StripeAPIChangeLogBridge.php | 23 + bridges/SuperbWallpapersBridge.php | 70 ++ bridges/T411Bridge.php | 96 ++ bridges/TagBoardBridge.php | 49 + bridges/TbibBridge.php | 12 + bridges/TheCodingLoveBridge.php | 46 + bridges/TheHackerNewsBridge.php | 80 ++ bridges/ThePirateBayBridge.php | 174 +++ bridges/TheTVDBBridge.php | 205 ++++ bridges/Torrent9Bridge.php | 102 ++ bridges/TwitterBridge.php | 287 +++++ bridges/UnsplashBridge.php | 77 ++ bridges/UsbekEtRicaBridge.php | 110 ++ bridges/ViadeoCompanyBridge.php | 37 + bridges/VineBridge.php | 40 + bridges/VkBridge.php | 66 ++ bridges/WallpaperStopBridge.php | 107 ++ bridges/WeLiveSecurityBridge.php | 45 + bridges/WebfailBridge.php | 149 +++ bridges/WhydBridge.php | 56 + bridges/WikiLeaksBridge.php | 129 +++ bridges/WikipediaBridge.php | 304 ++++++ bridges/WordPressBridge.php | 76 ++ bridges/WordPressPluginUpdateBridge.php | 87 ++ bridges/WorldOfTanksBridge.php | 72 ++ bridges/XbooruBridge.php | 12 + bridges/YandereBridge.php | 11 + bridges/YoutubeBridge.php | 189 ++++ bridges/ZDNetBridge.php | 302 ++++++ cache/.gitkeep | 0 caches/FileCache.php | 119 ++ formats/AtomFormat.php | 93 ++ formats/HtmlFormat.php | 100 ++ formats/JsonFormat.php | 25 + formats/MrssFormat.php | 106 ++ formats/PlaintextFormat.php | 25 + index.php | 268 +++++ lib/Bridge.php | 105 ++ lib/BridgeAbstract.php | 271 +++++ lib/BridgeInterface.php | 71 ++ lib/Cache.php | 53 + lib/CacheInterface.php | 7 + lib/Exceptions.php | 209 ++++ lib/FeedExpander.php | 208 ++++ lib/Format.php | 73 ++ lib/FormatAbstract.php | 106 ++ lib/FormatInterface.php | 11 + lib/RssBridge.php | 53 + lib/contents.php | 132 +++ lib/error.php | 28 + lib/html.php | 327 ++++++ lib/validation.php | 100 ++ phpcs.xml | 73 ++ scalingo.json | 6 + static/HtmlFormat.css | 113 ++ static/search.js | 27 + static/style.css | 280 +++++ vendor/simplehtmldom/simple_html_dom.php | 1742 ++++++++++++++++++++++++++++++ 174 files changed, 17225 insertions(+) create mode 100644 .gitattributes create mode 100644 .gitignore create mode 100644 .travis.yml create mode 100644 CHANGELOG.md create mode 100644 CONTRIBUTING.md create mode 100644 README.md create mode 100644 UNLICENSE create mode 100644 bridges/ABCTabsBridge.php create mode 100644 bridges/AcrimedBridge.php create mode 100644 bridges/AllocineFRBridge.php create mode 100644 bridges/AmazonBridge.php create mode 100644 bridges/AnimeUltimeBridge.php create mode 100644 bridges/Arte7Bridge.php create mode 100644 bridges/AskfmBridge.php create mode 100644 bridges/BandcampBridge.php create mode 100644 bridges/BastaBridge.php create mode 100644 bridges/BlaguesDeMerdeBridge.php create mode 100644 bridges/BooruprojectBridge.php create mode 100644 bridges/CADBridge.php create mode 100644 bridges/CNETBridge.php create mode 100644 bridges/CastorusBridge.php create mode 100644 bridges/CollegeDeFranceBridge.php create mode 100644 bridges/CommonDreamsBridge.php create mode 100644 bridges/CopieDoubleBridge.php create mode 100644 bridges/CourrierInternationalBridge.php create mode 100644 bridges/CpasbienBridge.php create mode 100644 bridges/CryptomeBridge.php create mode 100644 bridges/DailymotionBridge.php create mode 100644 bridges/DanbooruBridge.php create mode 100644 bridges/DansTonChatBridge.php create mode 100644 bridges/DauphineLibereBridge.php create mode 100644 bridges/DemoBridge.php create mode 100644 bridges/DeveloppezDotComBridge.php create mode 100644 bridges/DiceBridge.php create mode 100644 bridges/DilbertBridge.php create mode 100644 bridges/DollbooruBridge.php create mode 100644 bridges/DuckDuckGoBridge.php create mode 100644 bridges/EZTVBridge.php create mode 100644 bridges/EliteDangerousGalnetBridge.php create mode 100644 bridges/ElsevierBridge.php create mode 100644 bridges/EstCeQuonMetEnProdBridge.php create mode 100644 bridges/EtsyBridge.php create mode 100644 bridges/FB2Bridge.php create mode 100644 bridges/FacebookBridge.php create mode 100644 bridges/FeedExpanderExampleBridge.php create mode 100644 bridges/FierPandaBridge.php create mode 100644 bridges/FilterBridge.php create mode 100644 bridges/FlickrBridge.php create mode 100644 bridges/FootitoBridge.php create mode 100644 bridges/FourchanBridge.php create mode 100644 bridges/FuturaSciencesBridge.php create mode 100644 bridges/GBAtempBridge.php create mode 100644 bridges/GelbooruBridge.php create mode 100644 bridges/GiphyBridge.php create mode 100644 bridges/GithubIssueBridge.php create mode 100644 bridges/GithubSearchBridge.php create mode 100644 bridges/GizmodoBridge.php create mode 100644 bridges/GoComicsBridge.php create mode 100644 bridges/GooglePlusPostBridge.php create mode 100644 bridges/GoogleSearchBridge.php create mode 100644 bridges/HDWallpapersBridge.php create mode 100644 bridges/HentaiHavenBridge.php create mode 100644 bridges/IdenticaBridge.php create mode 100644 bridges/InstagramBridge.php create mode 100644 bridges/IsoHuntBridge.php create mode 100644 bridges/JapanExpoBridge.php create mode 100644 bridges/KATBridge.php create mode 100644 bridges/KernelBugTrackerBridge.php create mode 100644 bridges/KonachanBridge.php create mode 100644 bridges/KoreusBridge.php create mode 100644 bridges/KununuBridge.php create mode 100644 bridges/LWNprevBridge.php create mode 100755 bridges/LeBonCoinBridge.php create mode 100644 bridges/LeMondeInformatiqueBridge.php create mode 100644 bridges/LegifranceJOBridge.php create mode 100644 bridges/LesJoiesDuCodeBridge.php create mode 100644 bridges/LichessBridge.php create mode 100644 bridges/LinkedInCompanyBridge.php create mode 100644 bridges/LolibooruBridge.php create mode 100644 bridges/MangareaderBridge.php create mode 100644 bridges/MilbooruBridge.php create mode 100644 bridges/MixCloudBridge.php create mode 100644 bridges/MoebooruBridge.php create mode 100644 bridges/MoinMoinBridge.php create mode 100644 bridges/MondeDiploBridge.php create mode 100644 bridges/MsnMondeBridge.php create mode 100644 bridges/MspabooruBridge.php create mode 100644 bridges/NasaApodBridge.php create mode 100644 bridges/NeuviemeArtBridge.php create mode 100644 bridges/NextInpactBridge.php create mode 100644 bridges/NextgovBridge.php create mode 100644 bridges/NiceMatinBridge.php create mode 100644 bridges/NovelUpdatesBridge.php create mode 100644 bridges/OpenClassroomsBridge.php create mode 100644 bridges/ParuVenduImmoBridge.php create mode 100644 bridges/PickyWallpapersBridge.php create mode 100644 bridges/PinterestBridge.php create mode 100644 bridges/PlanetLibreBridge.php create mode 100644 bridges/RTBFBridge.php create mode 100644 bridges/RainbowSixSiegeBridge.php create mode 100644 bridges/ReadComicsBridge.php create mode 100644 bridges/Releases3DSBridge.php create mode 100644 bridges/ReporterreBridge.php create mode 100644 bridges/Rue89Bridge.php create mode 100644 bridges/Rule34Bridge.php create mode 100644 bridges/Rule34pahealBridge.php create mode 100644 bridges/SafebooruBridge.php create mode 100644 bridges/SakugabooruBridge.php create mode 100644 bridges/ScmbBridge.php create mode 100644 bridges/ScoopItBridge.php create mode 100644 bridges/SensCritiqueBridge.php create mode 100644 bridges/SexactuBridge.php create mode 100644 bridges/ShanaprojectBridge.php create mode 100644 bridges/Shimmie2Bridge.php create mode 100644 bridges/SoundcloudBridge.php create mode 100644 bridges/SteamBridge.php create mode 100644 bridges/StripeAPIChangeLogBridge.php create mode 100644 bridges/SuperbWallpapersBridge.php create mode 100644 bridges/T411Bridge.php create mode 100644 bridges/TagBoardBridge.php create mode 100644 bridges/TbibBridge.php create mode 100644 bridges/TheCodingLoveBridge.php create mode 100644 bridges/TheHackerNewsBridge.php create mode 100644 bridges/ThePirateBayBridge.php create mode 100644 bridges/TheTVDBBridge.php create mode 100644 bridges/Torrent9Bridge.php create mode 100644 bridges/TwitterBridge.php create mode 100644 bridges/UnsplashBridge.php create mode 100644 bridges/UsbekEtRicaBridge.php create mode 100644 bridges/ViadeoCompanyBridge.php create mode 100644 bridges/VineBridge.php create mode 100644 bridges/VkBridge.php create mode 100644 bridges/WallpaperStopBridge.php create mode 100644 bridges/WeLiveSecurityBridge.php create mode 100644 bridges/WebfailBridge.php create mode 100644 bridges/WhydBridge.php create mode 100644 bridges/WikiLeaksBridge.php create mode 100644 bridges/WikipediaBridge.php create mode 100644 bridges/WordPressBridge.php create mode 100644 bridges/WordPressPluginUpdateBridge.php create mode 100644 bridges/WorldOfTanksBridge.php create mode 100644 bridges/XbooruBridge.php create mode 100644 bridges/YandereBridge.php create mode 100644 bridges/YoutubeBridge.php create mode 100644 bridges/ZDNetBridge.php create mode 100644 cache/.gitkeep create mode 100644 caches/FileCache.php create mode 100644 formats/AtomFormat.php create mode 100644 formats/HtmlFormat.php create mode 100644 formats/JsonFormat.php create mode 100644 formats/MrssFormat.php create mode 100644 formats/PlaintextFormat.php create mode 100644 index.php create mode 100644 lib/Bridge.php create mode 100644 lib/BridgeAbstract.php create mode 100644 lib/BridgeInterface.php create mode 100644 lib/Cache.php create mode 100644 lib/CacheInterface.php create mode 100644 lib/Exceptions.php create mode 100644 lib/FeedExpander.php create mode 100644 lib/Format.php create mode 100644 lib/FormatAbstract.php create mode 100644 lib/FormatInterface.php create mode 100644 lib/RssBridge.php create mode 100644 lib/contents.php create mode 100644 lib/error.php create mode 100644 lib/html.php create mode 100644 lib/validation.php create mode 100644 phpcs.xml create mode 100644 scalingo.json create mode 100644 static/HtmlFormat.css create mode 100644 static/search.js create mode 100644 static/style.css create mode 100644 vendor/simplehtmldom/simple_html_dom.php diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..412eeda --- /dev/null +++ b/.gitattributes @@ -0,0 +1,22 @@ +# Auto detect text files and perform LF normalization +* text=auto + +# Custom for Visual Studio +*.cs diff=csharp +*.sln merge=union +*.csproj merge=union +*.vbproj merge=union +*.fsproj merge=union +*.dbproj merge=union + +# Standard to msysgit +*.doc diff=astextplain +*.DOC diff=astextplain +*.docx diff=astextplain +*.DOCX diff=astextplain +*.dot diff=astextplain +*.DOT diff=astextplain +*.pdf diff=astextplain +*.PDF diff=astextplain +*.rtf diff=astextplain +*.RTF diff=astextplain diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..d28d273 --- /dev/null +++ b/.gitignore @@ -0,0 +1,234 @@ +################# +## Eclipse +################# +vendor/* +data/ +*.pydevproject +.project +.metadata +bin/ +tmp/ +*.tmp +*.bak +*.swp +*~.nib +local.properties +.classpath +.settings/ +.loadpath + +# External tool builders +.externalToolBuilders/ + +# Locally stored "Eclipse launch configurations" +*.launch + +# CDT-specific +.cproject + +# PDT-specific +.buildpath + + +################# +## Visual Studio +################# + +## Ignore Visual Studio temporary files, build results, and +## files generated by popular Visual Studio add-ons. + +# User-specific files +*.suo +*.user +*.sln.docstates + +# Build results + +[Dd]ebug/ +[Rr]elease/ +x64/ +build/ +[Bb]in/ +[Oo]bj/ + +# MSTest test Results +[Tt]est[Rr]esult*/ +[Bb]uild[Ll]og.* + +*_i.c +*_p.c +*.ilk +*.meta +*.obj +*.pch +*.pdb +*.pgc +*.pgd +*.rsp +*.sbr +*.tlb +*.tli +*.tlh +*.tmp +*.tmp_proj +*.log +*.vspscc +*.vssscc +.builds +*.pidb +*.log +*.scc + +# Visual C++ cache files +ipch/ +*.aps +*.ncb +*.opensdf +*.sdf +*.cachefile + +# Visual Studio profiler +*.psess +*.vsp +*.vspx + +# Guidance Automation Toolkit +*.gpState + +# ReSharper is a .NET coding add-in +_ReSharper*/ +*.[Rr]e[Ss]harper + +# TeamCity is a build add-in +_TeamCity* + +# DotCover is a Code Coverage Tool +*.dotCover + +# NCrunch +*.ncrunch* +.*crunch*.local.xml + +# Installshield output folder +[Ee]xpress/ + +# DocProject is a documentation generator add-in +DocProject/buildhelp/ +DocProject/Help/*.HxT +DocProject/Help/*.HxC +DocProject/Help/*.hhc +DocProject/Help/*.hhk +DocProject/Help/*.hhp +DocProject/Help/Html2 +DocProject/Help/html + +# Click-Once directory +publish/ + +# Publish Web Output +*.Publish.xml +*.pubxml + +# NuGet Packages Directory +## TODO: If you have NuGet Package Restore enabled, uncomment the next line +#packages/ + +# Windows Azure Build Output +csx +*.build.csdef + +# Windows Store app package directory +AppPackages/ + +# Others +sql/ +*.Cache +ClientBin/ +[Ss]tyle[Cc]op.* +~$* +*~ +*.dbmdl +*.[Pp]ublish.xml +*.pfx +*.publishsettings + +# RIA/Silverlight projects +Generated_Code/ + +# Backup & report files from converting an old project file to a newer +# Visual Studio version. Backup files are not needed, because we have git ;-) +_UpgradeReport_Files/ +Backup*/ +UpgradeLog*.XML +UpgradeLog*.htm + +# SQL Server files +App_Data/*.mdf +App_Data/*.ldf + +################# +## Other ide stuff +################# +.idea/* +[#]*[#] + +############# +## Windows detritus +############# + +# Windows image file caches +Thumbs.db +ehthumbs.db + +# Folder config file +Desktop.ini + +# Recycle Bin used on file shares +$RECYCLE.BIN/ + +# Mac crap +.DS_Store + + +############# +## Python +############# + +*.py[co] + +# Packages +*.egg +*.egg-info +dist/ +build/ +eggs/ +parts/ +var/ +sdist/ +develop-eggs/ +.installed.cfg + +# Installer logs +pip-log.txt + +# Unit test / coverage reports +.coverage +.tox + +#Translations +*.mo + +#Mr Developer +.mr.developer.cfg + +############## +## RSS-Bridge +############## +/cache +/whitelist.txt +DEBUG + +###################### +## VisualStudioCode ## +###################### +.vscode/* \ No newline at end of file diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..969c1b3 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,19 @@ +language: php +php: + - '5.6' + - '7.0' + - hhvm + - nightly + +install: + - pear install PHP_CodeSniffer + +script: + - phpenv rehash + - phpcs . --standard=phpcs.xml --warning-severity=0 --extensions=php -p + +matrix: + fast_finish: true + allow_failures: + - php: hhvm + - php: nightly \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..4f589ff --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,233 @@ +rss-bridge Changelog +=== + +RSS-Bridge 2017-08-03 +== + +## Important changes +RSS-Bridge now has [contribution guidelines](CONTRIBUTING.md) +[phpcs rules](phpcs.xml) follow the [contribution guidelines](CONTRIBUTING.md) + +## General changes +Added a search bar to make searching for bridges easier +Added user friendly error page for when a bridge fails +Added caching of extraInfos (name, uri) +Added an indicator to warn for bridges using HTTP instead of HTTPS +Various bug fixes and improvements + +## Modified bridges +[AllocineFRBridge] Update Faux Raccord link +[DanbooruBridge] Fix broken URI +[DuckDuckGoBridge] Disable DuckDuckGo redirects so that the links returned are correct. +[FacebookBridge] Add option to hide posts with facebook videos +[FacebookBridge] Add requester languages to HTTP header +[FacebookBridge] Handle summary posts +[FacebookBridge] Replace 'novideo' with 'media_type' +[FilterBridge] Initial implementation of basic title permit and block +[FlickrTagBridge] Fix and improve bridge by using the FlickrExploreBridge approach +[GooglePlusPostBridge] Autofix user names +[GooglePlusPostBridge] Fix bridge implementation +[GooglePlusPostBridge] Fix content loading +[InstagramBridge] Add option to filter for videos and pictures +[LWNprevBridge] full rewrite +[MangareaderBridge] Fix double forward slashes +[NasaApodBridge] Use HTTPS instead of HTTP +[PinterestBridge] Fix checkbox not working +[PinterestBridge] Fix implementation after DOM changes +[RTBFBridge] Update URI +[SexactuBridge] Fix URI and timestamp +[SexactuBridge] Use most modern version of bridge api and cached pages (#504) +[ShanaprojectBridge] Don't throw error if timestamp is missing +[TwitterBridge] Add option to hide retweets +[TwitterBridge] Avoid empty content caused by new login policy +[TwitterBridge] Fix double slashes in URI +[TwitterBridge] Fix missing spaces +[TwitterBridge] Fix title includes anchors in plaintext format +[TwitterBridge] ignore promoted tweets +[TwitterBridge] Optimize returned image sizes +[TwitterBridge] Show quotes and pictures +[WebfailBridge] Properly handle gifs (DOM changed) +[YoutubeBridge] Improve readability of feed contents +[YoutubeBridge] Improve URL handling in video descriptions + +## New bridges +AmazonBridge +DiceBridge +EtsyBridge +FB2Bridge +FilterBridge +FlickrBridge +GithubSearchBridge +GoComicsBridge +KATBridge +KernelBugTrackerBridge +MixCloudBridge +MoinMoinBridge +RainbowSixSiegeBridge +SteamBridge +TheTVDBBridge +Torrent9Bridge +UsbekEtRicaBridge +WikiLeaksBridge +WordPressPluginUpdateBridge + +Alpha 0.2 +=== + +## Important changes +* RSS-Bridge has been [UNLICENSED](UNLICENSE) +* RSS-Bridge is now a community-managed project on [GitHub](https://github.com/rss-bridge/rss-bridge) +* RSS-Bridge now has a [Wiki](https://github.com/rss-bridge/rss-bridge/wiki) +* RSS-Bridge now supports [Travis-CI](https://travis-ci.org) + +## General changes +* Added [CHANGELOG](CHANGELOG.md) (this file) +* Added [PHP Simple HTML DOM Parser](http://simplehtmldom.sourceforge.net) to [vendor](vendor/simplehtmldom/) +* Added cache purging function (cache will be force-purged after 24 hours or as defined by bridge) +* Added new format [MrssFormat](formats/MrssFormat.php) +* Added parameter `author` - for display of the feed author name - to all formats +* Added new abstraction of the BridgeInterface: + - [FeedExpander](https://github.com/RSS-Bridge/rss-bridge/wiki/Bridge-API) +* Added optional support for proxy usage on each individual bridge +* Added support for [custom bridge parameter](https://github.com/RSS-Bridge/rss-bridge/wiki/BridgeAbstract#format-specifications) (text, number, list, checkbox) +* Changed design of the welcome screen +* Changed design of HtmlFormat +* Changed behavior of debug mode: + - Enable debug mode by placing a file called "DEBUG" in the root folder + - Debug mode automatically disables cache file loading +* Changed implementation of bridges - see [Wiki](https://github.com/rss-bridge/rss-bridge/wiki) + - Changed comment-style metadata to constants + - Added support for multiple utilizations per bridge + - Changed the parameter loading algorithm to be loaded by RSS-Bridge core +* Improved checks for PHP version, configuration and extensions +* Many bug fixes + +## Modified Bridges +* FlickrExploreBridge +* GoogleSearchBridge +* TwitterBridge + +## New Bridges +* ABCTabsBridge +* AcrimedBridge +* AllocineFRBridge +* AnimeUltimeBridge +* Arte7Bridge +* AskfmBridge +* BandcampBridge +* BastaBridge +* BlaguesDeMerdeBridge +* BooruprojectBridge +* CADBridge +* CNETBridge +* CastorusBridge +* CollegeDeFranceBridge +* CommonDreamsBridge +* CopieDoubleBridge +* CourrierInternationalBridge +* CpasbienBridge +* CryptomeBridge +* DailymotionBridge +* DanbooruBridge +* DansTonChatBridge +* DauphineLibereBridge +* DemoBridge +* DeveloppezDotComBridge +* DilbertBridge +* DollbooruBridge +* DuckDuckGoBridge +* EZTVBridge +* EliteDangerousGalnetBridge +* ElsevierBridge +* EstCeQuonMetEnProdBridge +* FacebookBridge +* FierPandaBridge +* FlickrTagBridge +* FootitoBridge +* FourchanBridge +* FuturaSciencesBridge +* GBAtempBridge +* GelbooruBridge +* GiphyBridge +* GithubIssueBridge +* GizmodoBridge +* GooglePlusPostBridge +* HDWallpapersBridge +* HentaiHavenBridge +* IdenticaBridge +* InstagramBridge +* IsoHuntBridge +* JapanExpoBridge +* KonachanBridge +* KoreusBridge +* KununuBridge +* LWNprevBridge +* LeBonCoinBridge +* LegifranceJOBridge +* LeMondeInformatiqueBridge +* LesJoiesDuCodeBridge +* LichessBridge +* LinkedInCompanyBridge +* LolibooruBridge +* MangareaderBridge +* MilbooruBridge +* MoebooruBridge +* MondeDiploBridge +* MsnMondeBridge +* MspabooruBridge +* NasaApodBridge +* NeuviemeArtBridge +* NextInpactBridge +* NextgovBridge +* NiceMatinBridge +* NovelUpdatesBridge +* OpenClassroomsBridge +* ParuVenduImmoBridge +* PickyWallpapersBridge +* PinterestBridge +* PlanetLibreBridge +* RTBFBridge +* ReadComicsBridge +* Releases3DSBridge +* ReporterreBridge +* Rue89Bridge +* Rule34Bridge +* Rule34pahealBridge +* SafebooruBridge +* SakugabooruBridge +* ScmbBridge +* ScoopItBridge +* SensCritiqueBridge +* SexactuBridge +* ShanaprojectBridge +* Shimmie2Bridge +* SoundcloudBridge +* StripeAPIChangeLogBridge +* SuperbWallpapersBridge +* T411Bridge +* TagBoardBridge +* TbibBridge +* TheCodingLoveBridge +* TheHackerNewsBridge +* ThePirateBayBridge +* UnsplashBridge +* ViadeoCompanyBridge +* VineBridge +* VkBridge +* WallpaperStopBridge +* WebfailBridge +* WeLiveSecurityBridge +* WhydBridge +* WikipediaBridge +* WordPressBridge +* WorldOfTanksBridge +* XbooruBridge +* YandereBridge +* YoutubeBridge +* ZDNetBridge + +Alpha 0.1 +=== +* First tagged version. +* Includes refactoring. +* Unstable. \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..e03f926 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,47 @@ +### Pull request policy +Fix one issue per pull request. +Squash commits before opening a pull request. +Respect the coding style policy. +Name your PR like the following : + +* When correcting a single bridge, use `[BridgeName] Feature`. +* When fixing a problem in a specific file, use `[FileName] Feature`. +* When fixing a general problem, use `category : feature`. + +Note that all pull-requests should pass the unit tests before they can be merged. + +### Coding style + +Use `camelCase` for variables and methods. +Use `UPPERCASE` for constants. +Use `PascalCase` for class names. When creating a bridge, your class and PHP file should be named `MyImplementationBridge`. +Use tabs for indentation. +Add an empty line at the end of your file. + +Use `''` to encapsulate strings, including in arrays. +Prefer lines shorter than 80 chars, no line longer than 120 chars. +PHP constants should be in lower case (`true, false, null`...) + + +* Add spaces between the logical operator and your expressions (not needed for the `!` operator). +* Use `||` and `&&` instead of `or` and `and`. +* Add space between your condition and the opening bracket/closing bracket. +* Don't put a space between `if` and your bracket. +* Use `elseif` instead of `else if`. +* Add new lines in your conditions if they are containing more than one line. +* Example : + +```PHP +if($a == true && $b) { + print($a); +} else if(!$b) { + + $a = !$a; + $b = $b >> $a; + print($b); + +} else { + print($b); +} +``` + diff --git a/README.md b/README.md new file mode 100644 index 0000000..e36f22b --- /dev/null +++ b/README.md @@ -0,0 +1,142 @@ +rss-bridge +=== +[![LICENSE](https://img.shields.io/badge/license-UNLICENSE-blue.svg)](UNLICENSE) [![Build Status](https://travis-ci.org/RSS-Bridge/rss-bridge.svg?branch=master)](https://travis-ci.org/RSS-Bridge/rss-bridge) + +rss-bridge is a PHP project capable of generating ATOM feeds for websites which don't have one. + +Supported sites/pages (main) +=== + + * `FlickrExplore` : [Latest interesting images](http://www.flickr.com/explore) from Flickr + * `GoogleSearch` : Most recent results from Google Search + * `GooglePlus` : Most recent posts of user timeline + * `Twitter` : Return keyword/hashtag search or user timeline + * `Identi.ca` : Identica user timeline (Should be compatible with other Pump.io instances) + * `YouTube` : YouTube user channel, playlist or search + * `Cryptome` : Returns the most recent documents from [Cryptome.org](http://cryptome.org/) + * `DansTonChat`: Most recent quotes from [danstonchat.com](http://danstonchat.com/) + * `DuckDuckGo`: Most recent results from [DuckDuckGo.com](https://duckduckgo.com/) + * `Instagram`: Most recent photos from an Instagram user + * `OpenClassrooms`: Lastest tutorials from [fr.openclassrooms.com](http://fr.openclassrooms.com/) + * `Pinterest`: Most recent photos from user or search + * `ScmbBridge`: Newest stories from [secouchermoinsbete.fr](http://secouchermoinsbete.fr/) + * `Wikipedia`: highlighted articles from [Wikipedia](https://wikipedia.org/) in English, German, French or Esperanto + * `Bandcamp` : Returns last release from [bandcamp](https://bandcamp.com/) for a tag + * `ThePirateBay` : Returns the newest indexed torrents from [The Pirate Bay](https://thepiratebay.se/) with keywords + * `Facebook` : Returns the latest posts on a page or profile on [Facebook](https://facebook.com/) + +Plus [many other bridges](bridges/) to enable, thanks to the community + +Output format +=== +Output format can take several forms: + + * `Atom` : ATOM Feed, for use in RSS/Feed readers + * `Mrss` : MRSS Feed, for use in RSS/Feed readers + * `Json` : Json, for consumption by other applications. + * `Html` : Simple html page. + * `Plaintext` : raw text (php object, as returned by print_r) + +Screenshot +=== + +Welcome screen: + +![Screenshot](https://github.com/RSS-Bridge/rss-bridge/wiki/images/screenshot_rss-bridge_welcome.png) + +RSS-Bridge hashtag (#rss-bridge) search on Twitter, in ATOM format (as displayed by Firefox): + +![Screenshot](https://github.com/RSS-Bridge/rss-bridge/wiki/images/screenshot_twitterbridge_atom.png) + +Requirements +=== + + * PHP 5.6, e.g. `AddHandler application/x-httpd-php56 .php` in `.htaccess` + * `openssl` extension enabled in PHP config (`php.ini`) + * `allow_url_fopen=1` in `php.ini` + +Enabling/Disabling bridges +=== + +By default, the script creates `whitelist.txt` and adds the main bridges (see above). `whitelist.txt` is ignored by git, you can edit it: + * to enable extra bridges (one bridge per line) + * to disable main bridges (remove the line) + * to enable all bridges (just one wildcard `*` as file content) + +New bridges are disabled by default, so make sure to check regularly what's new and whitelist what you want! + +Deploy +=== +[![Deploy on Scalingo](https://cdn.scalingo.com/deploy/button.svg)](https://my.scalingo.com/deploy?source=https://github.com/sebsauvage/rss-bridge) + +Authors +=== +We are RSS Bridge Community, a group of developers continuing the project initiated by sebsauvage, webmaster of [sebsauvage.net](http://sebsauvage.net), author of [Shaarli](http://sebsauvage.net/wiki/doku.php?id=php:shaarli) and [ZeroBin](http://sebsauvage.net/wiki/doku.php?id=php:zerobin). + +Patch/contributors : + + * Yves ASTIER ([Draeli](https://github.com/Draeli)) : PHP optimizations, fixes, dynamic brigde/format list with all stuff behind and extend cache system. Mail : contact /at\ yves-astier.com + * [Mitsukarenai](https://github.com/Mitsukarenai) : Initial inspiration, collaborator + * [ArthurHoaro](https://github.com/ArthurHoaro) + * [BoboTiG](https://github.com/BoboTiG) + * [Astalaseven](https://github.com/Astalaseven) + * [qwertygc](https://github.com/qwertygc) + * [Djuuu](https://github.com/Djuuu) + * [Anadrark](https://github.com/Anadrark]) + * [Grummfy](https://github.com/Grummfy) + * [Polopollo](https://github.com/Polopollo) + * [16mhz](https://github.com/16mhz) + * [kranack](https://github.com/kranack) + * [logmanoriginal](https://github.com/logmanoriginal) + * [polo2ro](https://github.com/polo2ro) + * [Riduidel](https://github.com/Riduidel) + * [superbaillot.net](http://superbaillot.net/) + * [vinzv](https://github.com/vinzv) + * [teromene](https://github.com/teromene) + * [nel50n](https://github.com/nel50n) + * [nyutag](https://github.com/nyutag) + * [ORelio](https://github.com/ORelio) + * [Pitchoule](https://github.com/Pitchoule) + * [pit-fgfjiudghdf](https://github.com/pit-fgfjiudghdf) + * [aledeg](https://github.com/aledeg) + * [alexAubin](https://github.com/alexAubin) + * [cnlpete](https://github.com/cnlpete) + * [corenting](https://github.com/corenting) + * [Daiyousei](https://github.com/Daiyousei) + * [erwang](https://github.com/erwang) + * [gsurrel](https://github.com/gsurrel) + * [kraoc](https://github.com/kraoc) + * [lagaisse](https://github.com/lagaisse) + * [az5he6ch](https://github.com/az5he6ch) + * [niawag](https://github.com/niawag) + * [JeremyRand](https://github.com/JeremyRand) + * [mro](https://github.com/mro) + +Licenses +=== +Code is [Public Domain](UNLICENSE). + +Including `PHP Simple HTML DOM Parser` under the [MIT License](http://opensource.org/licenses/MIT) + + +Technical notes +=== + * There is a cache so that source services won't ban you even if you hammer the rss-bridge with requests. Each bridge can have a different duration for the cache. The `cache` subdirectory will be automatically created and cached objects older than 24 hours get purged. + * To implement a new Bridge, [follow the specifications](https://github.com/RSS-Bridge/rss-bridge/wiki/Bridge-API) and take a look at existing Bridges for examples. + * To enable debug mode (disabling cache and enabling error reporting), create an empty file named `DEBUG` in the root directory (next to `index.php`). + * For more information refer to the [Wiki](https://github.com/RSS-Bridge/rss-bridge/wiki) + +Rant +=== + +*Dear so-called "social" websites.* + +Your catchword is "share", but you don't want us to share. You want to keep us within your walled gardens. That's why you've been removing RSS links from webpages, hiding them deep on your website, or removed feeds entirely, replacing it with crippled or demented proprietary API. **FUCK YOU.** + +You're not social when you hamper sharing by removing feeds. You're happy to have customers creating content for your ecosystem, but you don't want this content out - a content you do not even own. Google Takeout is just a gimmick. We want our data to flow, we want RSS or ATOM feeds. + +We want to share with friends, using open protocols: RSS, ATOM, XMPP, whatever. Because no one wants to have *your* service with *your* applications using *your* API force-feeding them. Friends must be free to choose whatever software and service they want. + +We are rebuilding bridges you have wilfully destroyed. + +Get your shit together: Put RSS/ATOM back in. diff --git a/UNLICENSE b/UNLICENSE new file mode 100644 index 0000000..a84c395 --- /dev/null +++ b/UNLICENSE @@ -0,0 +1,25 @@ +This is free and unencumbered software released into the public domain. + +Anyone is free to copy, modify, publish, use, compile, sell, or +distribute this software, either in source code form or as a compiled +binary, for any purpose, commercial or non-commercial, and by any +means. + +In jurisdictions that recognize copyright laws, the author or authors +of this software dedicate any and all copyright interest in the +software to the public domain. We make this dedication for the benefit +of the public at large and to the detriment of our heirs and +successors. We intend this dedication to be an overt act of +relinquishment in perpetuity of all present and future rights to this +software under copyright law. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +OTHER DEALINGS IN THE SOFTWARE. + +For more information, please refer to + diff --git a/bridges/ABCTabsBridge.php b/bridges/ABCTabsBridge.php new file mode 100644 index 0000000..2e451e2 --- /dev/null +++ b/bridges/ABCTabsBridge.php @@ -0,0 +1,42 @@ +find('table#myTable', 0)->children(1); + + foreach ($table->find('tr') as $tab) { + $item = array(); + $item['author'] = $tab->find('td', 1)->plaintext + . ' - ' + . $tab->find('td', 2)->plaintext; + + $item['title'] = $tab->find('td', 1)->plaintext + . ' - ' + . $tab->find('td', 2)->plaintext; + + $item['content'] = 'Le ' + . $tab->find('td', 0)->plaintext + . '
Par: ' + . $tab->find('td', 5)->plaintext + . '
Type: ' + . $tab->find('td', 3)->plaintext; + + $item['id'] = static::URI + . $tab->find('td', 2)->find('a', 0)->getAttribute('href'); + + $item['uri'] = static::URI + . $tab->find('td', 2)->find('a', 0)->getAttribute('href'); + + $this->items[] = $item; + } + } +} diff --git a/bridges/AcrimedBridge.php b/bridges/AcrimedBridge.php new file mode 100644 index 0000000..8b40d1d --- /dev/null +++ b/bridges/AcrimedBridge.php @@ -0,0 +1,25 @@ +collectExpandableDatas(static::URI . 'spip.php?page=backend'); + } + + protected function parseItem($newsItem){ + $item = parent::parseItem($newsItem); + + $articlePage = getSimpleHTMLDOM($newsItem->link); + $article = sanitize($articlePage->find('article.article1', 0)->innertext); + $article = defaultLinkTo($article, static::URI); + $item['content'] = $article; + + return $item; + } + +} diff --git a/bridges/AllocineFRBridge.php b/bridges/AllocineFRBridge.php new file mode 100644 index 0000000..604199b --- /dev/null +++ b/bridges/AllocineFRBridge.php @@ -0,0 +1,87 @@ + array( + 'name' => 'category', + 'type' => 'list', + 'required' => true, + 'exampleValue' => 'Faux Raccord', + 'title' => 'Select your category', + 'values' => array( + 'Faux Raccord' => 'faux-raccord', + 'Top 5' => 'top-5', + 'Tueurs en Séries' => 'tueurs-en-serie' + ) + ) + )); + + public function getURI(){ + if(!is_null($this->getInput('category'))) { + + switch($this->getInput('category')) { + case 'faux-raccord': + $uri = static::URI . 'video/programme-12284/saison-29841/'; + break; + case 'top-5': + $uri = static::URI . 'video/programme-12299/saison-29561/'; + break; + case 'tueurs-en-serie': + $uri = static::URI . 'video/programme-12286/saison-22938/'; + break; + } + + return $uri; + } + + return parent::getURI(); + } + + public function getName(){ + if(!is_null($this->getInput('category'))) { + return self::NAME . ' : ' + .array_search( + $this->getInput('category'), + self::PARAMETERS[$this->queriedContext]['category']['values'] + ); + } + + return parent::getName(); + } + + public function collectData(){ + + $html = getSimpleHTMLDOM($this->getURI()) + or returnServerError('Could not request ' . $this->getURI() . ' !'); + + $category = array_search( + $this->getInput('category'), + self::PARAMETERS[$this->queriedContext]['category']['values'] + ); + + foreach($html->find('figure.media-meta-fig') as $element) { + $item = array(); + + $title = $element->find('div.titlebar h3.title a', 0); + $content = trim($element->innertext); + $figCaption = strpos($content, $category); + + if($figCaption !== false) { + $content = str_replace('src="/', 'src="' . static::URI, $content); + $content = str_replace('href="/', 'href="' . static::URI, $content); + $content = str_replace('src=\'/', 'src=\'' . static::URI, $content); + $content = str_replace('href=\'/', 'href=\'' . static::URI, $content); + $item['content'] = $content; + $item['title'] = trim($title->innertext); + $item['uri'] = static::URI . $title->href; + $this->items[] = $item; + } + } + } + +} diff --git a/bridges/AmazonBridge.php b/bridges/AmazonBridge.php new file mode 100644 index 0000000..cbc6119 --- /dev/null +++ b/bridges/AmazonBridge.php @@ -0,0 +1,94 @@ + array( + 'name' => 'Keyword', + 'required' => true, + ), + 'sort' => array( + 'name' => 'Sort by', + 'type' => 'list', + 'required' => false, + 'values' => array( + 'Relevance' => 'relevanceblender', + 'Price: Low to High' => 'price-asc-rank', + 'Price: High to Low' => 'price-desc-rank', + 'Average Customer Review' => 'review-rank', + 'Newest Arrivals' => 'date-desc-rank', + ), + 'defaultValue' => 'relevanceblender', + ), + 'tld' => array( + 'name' => 'Country', + 'type' => 'list', + 'required' => true, + 'values' => array( + 'Australia' => 'com.au', + 'Brazil' => 'com.br', + 'Canada' => 'ca', + 'China' => 'cn', + 'France' => 'fr', + 'Germany' => 'de', + 'India' => 'in', + 'Italy' => 'it', + 'Japan' => 'co.jp', + 'Mexico' => 'com.mx', + 'Netherlands' => 'nl', + 'Spain' => 'es', + 'United Kingdom' => 'co.uk', + 'United States' => 'com', + ), + 'defaultValue' => 'com', + ), + )); + + public function getName(){ + if(!is_null($this->getInput('tld')) && !is_null($this->getInput('q'))) { + return 'Amazon.'.$this->getInput('tld').': '.$this->getInput('q'); + } + + return parent::getName(); + } + + public function collectData() { + + $uri = 'https://www.amazon.'.$this->getInput('tld').'/'; + $uri .= 's/?field-keywords='.urlencode($this->getInput('q')).'&sort='.$this->getInput('sort'); + + $html = getSimpleHTMLDOM($uri) + or returnServerError('Could not request Amazon.'); + + foreach($html->find('li.s-result-item') as $element) { + + $item = array(); + + // Title + $title = $element->find('h2', 0); + + $item['title'] = html_entity_decode($title->innertext, ENT_QUOTES); + + // Url + $uri = $title->parent()->getAttribute('href'); + $uri = substr($uri, 0, strrpos($uri, '/')); + + $item['uri'] = substr($uri, 0, strrpos($uri, '/')); + + // Content + $image = $element->find('img', 0); + $price = $element->find('span.s-price', 0); + $price = ($price) ? $price->innertext : ''; + + $item['content'] = '
'.$price; + + $this->items[] = $item; + } + } +} diff --git a/bridges/AnimeUltimeBridge.php b/bridges/AnimeUltimeBridge.php new file mode 100644 index 0000000..6c5427e --- /dev/null +++ b/bridges/AnimeUltimeBridge.php @@ -0,0 +1,135 @@ + array( + 'name' => 'Type', + 'type' => 'list', + 'values' => array( + 'Everything' => '', + 'Anime' => 'A', + 'Drama' => 'D', + 'Tokusatsu' => 'T' + ) + ) + )); + + private $filter = 'Releases'; + + public function collectData(){ + + //Add type filter if provided + $typeFilter = array_search( + $this->getInput('type'), + self::PARAMETERS[$this->queriedContext]['type']['values'] + ); + + //Build date and filters for making requests + $thismonth = date('mY') . $typeFilter; + $lastmonth = date('mY', mktime(0, 0, 0, date('n') - 1, 1, date('Y'))) . $typeFilter; + + //Process each HTML page until having 10 releases + $processedOK = 0; + foreach (array($thismonth, $lastmonth) as $requestFilter) { + + //Retrive page contents + $url = self::URI . 'history-0-1/' . $requestFilter; + $html = getSimpleHTMLDOM($url) + or returnServerError('Could not request Anime-Ultime: ' . $url); + + //Relases are sorted by day : process each day individually + foreach($html->find('div.history', 0)->find('h3') as $daySection) { + + //Retrieve day and build date information + $dateString = $daySection->plaintext; + $day = intval(substr($dateString, strpos($dateString, ' ') + 1, 2)); + $item_date = strtotime(str_pad($day, 2, '0', STR_PAD_LEFT) + . '-' + . substr($requestFilter, 0, 2) + . '-' + . substr($requestFilter, 2, 4)); + + //

day


<-- useful data in table rows + $release = $daySection->next_sibling()->next_sibling()->first_child(); + + //Process each release of that day, ignoring first table row: contains table headers + while(!is_null($release = $release->next_sibling())) { + if(count($release->find('td')) > 0) { + + //Retrieve metadata from table columns + $item_link_element = $release->find('td', 0)->find('a', 0); + $item_uri = self::URI . $item_link_element->href; + $item_name = html_entity_decode($item_link_element->plaintext); + $item_episode = html_entity_decode( + str_pad( + $release->find('td', 1)->plaintext, + 2, + '0', + STR_PAD_LEFT + ) + ); + + $item_fansub = $release->find('td', 2)->plaintext; + $item_type = $release->find('td', 4)->plaintext; + + if(!empty($item_uri)) { + + // Retrieve description from description page and + // convert relative image src info absolute image src + $html_item = getContents($item_uri) + or returnServerError('Could not request Anime-Ultime: ' . $item_uri); + $item_description = substr( + $html_item, + strpos($html_item, 'class="principal_contain" align="center">') + 41 + ); + $item_description = substr($item_description, + 0, + strpos($item_description, '
') + ); + $item_description = str_replace( + 'src="images', 'src="' . self::URI . 'images', + $item_description + ); + $item_description = str_replace("\r", '', $item_description); + $item_description = str_replace("\n", '', $item_description); + $item_description = utf8_encode($item_description); + + //Build and add final item + $item = array(); + $item['uri'] = $item_uri; + $item['title'] = $item_name . ' ' . $item_type . ' ' . $item_episode; + $item['author'] = $item_fansub; + $item['timestamp'] = $item_date; + $item['content'] = $item_description; + $this->items[] = $item; + $processedOK++; + + //Stop processing once limit is reached + if ($processedOK >= 10) + return; + } + } + } + } + } + } + + public function getName() { + if(!is_null($this->getInput('type'))) { + $typeFilter = array_search( + $this->getInput('type'), + self::PARAMETERS[$this->queriedContext]['type']['values'] + ); + + return 'Latest ' . $typeFilter . ' - Anime-Ultime Bridge'; + } + + return parent::getName(); + } + +} diff --git a/bridges/Arte7Bridge.php b/bridges/Arte7Bridge.php new file mode 100644 index 0000000..3d7ae9d --- /dev/null +++ b/bridges/Arte7Bridge.php @@ -0,0 +1,102 @@ + array( + 'catfr' => array( + 'type' => 'list', + 'name' => 'Catégorie', + 'values' => array( + 'Toutes les vidéos (français)' => 'toutes-les-videos', + 'Actu & société' => 'actu-société', + 'Séries & fiction' => 'séries-fiction', + 'Cinéma' => 'cinéma', + 'Arts & spectacles classiques' => 'arts-spectacles-classiques', + 'Culture pop' => 'culture-pop', + 'Découverte' => 'découverte', + 'Histoire' => 'histoire', + 'Junior' => 'junior' + ) + ) + ), + 'Catégorie (Allemand)' => array( + 'catde' => array( + 'type' => 'list', + 'name' => 'Catégorie', + 'values' => array( + 'Alle Videos (deutsch)' => 'alle-videos', + 'Aktuelles & Gesellschaft' => 'aktuelles-gesellschaft', + 'Fernsehfilme & Serien' => 'fernsehfilme-serien', + 'Kino' => 'kino', + 'Kunst & Kultur' => 'kunst-kultur', + 'Popkultur & Alternativ' => 'popkultur-alternativ', + 'Entdeckung' => 'entdeckung', + 'Geschichte' => 'geschichte', + 'Junior' => 'junior' + ) + ) + ) + ); + + public function collectData(){ + switch($this->queriedContext) { + case 'Catégorie (Français)': + $category = $this->getInput('catfr'); + $lang = 'fr'; + break; + case 'Catégorie (Allemand)': + $category = $this->getInput('catde'); + $lang = 'de'; + break; + } + + $url = self::URI . 'guide/' . $lang . '/plus7/' . $category; + $input = getContents($url) or die('Could not request ARTE.'); + + if(strpos($input, 'categoryVideoSet') !== false) { + $input = explode('categoryVideoSet="', $input); + $input = explode('}}', $input[1]); + $input = $input[0] . '}}'; + } else { + $input = explode('videoSet="', $input); + $input = explode('}]}', $input[1]); + $input = $input[0] . '}]}'; + } + + $input_json = json_decode(html_entity_decode($input, ENT_QUOTES), true); + + foreach($input_json['videos'] as $element) { + $item = array(); + $item['uri'] = str_replace("autoplay=1", "", $element['url']); + $item['id'] = $element['id']; + + $hack_broadcast_time = $element['rights_end']; + $hack_broadcast_time = strtok($hack_broadcast_time, 'T'); + $hack_broadcast_time = strtok('T'); + + $item['timestamp'] = strtotime($element['scheduled_on'] . 'T' . $hack_broadcast_time); + $item['title'] = $element['title']; + + if(!empty($element['subtitle'])) + $item['title'] = $element['title'] . ' | ' . $element['subtitle']; + + $item['duration'] = round((int)$element['duration'] / 60); + $item['content'] = $element['teaser'] + . '

' + . $item['duration'] + . 'min
'; + + $this->items[] = $item; + } + } + +} diff --git a/bridges/AskfmBridge.php b/bridges/AskfmBridge.php new file mode 100644 index 0000000..e227461 --- /dev/null +++ b/bridges/AskfmBridge.php @@ -0,0 +1,74 @@ + array( + 'u' => array( + 'name' => 'Username', + 'required' => true + ) + ) + ); + + public function collectData(){ + $html = getSimpleHTMLDOM($this->getURI()) + or returnServerError('Requested username can\'t be found.'); + + foreach($html->find('div.streamItem-answer') as $element) { + $item = array(); + $item['uri'] = self::URI . $element->find('a.streamItemsAge', 0)->href; + $question = trim($element->find('h1.streamItemContent-question', 0)->innertext); + + $item['title'] = trim( + htmlspecialchars_decode($element->find('h1.streamItemContent-question', 0)->plaintext, + ENT_QUOTES + ) + ); + + $answer = trim($element->find('p.streamItemContent-answer', 0)->innertext); + + // Doesn't work, DOM parser doesn't seem to like data-hint, dunno why + #$item['update'] = $element->find('a.streamitemsage',0)->data-hint; + + // This probably should be cleaned up, especially for YouTube embeds + $visual = $element->find('div.streamItemContent-visual', 0)->innertext; + //Fix tracking links, also doesn't work + foreach($element->find('a') as $link) { + if(strpos($link->href, 'l.ask.fm') !== false) { + + // Too slow + #$link->href = str_replace('#_=_', '', get_headers($link->href, 1)['Location']); + + $link->href = $link->plaintext; + } + } + + $content = '

' . $question . '

' . $answer . '

' . $visual . '

'; + // Fix relative links without breaking // scheme used by YouTube stuff + $content = preg_replace('#href="\/(?!\/)#', 'href="' . self::URI, $content); + $item['content'] = $content; + $this->items[] = $item; + } + } + + public function getName(){ + if(!is_null($this->getInput('u'))) { + return self::NAME . ' : ' . $this->getInput('u'); + } + + return parent::getName(); + } + + public function getURI(){ + if(!is_null($this->getInput('u'))) { + return self::URI . urlencode($this->getInput('u')) . '/answers/more?page=0'; + } + + return parent::getURI(); + } +} diff --git a/bridges/BandcampBridge.php b/bridges/BandcampBridge.php new file mode 100644 index 0000000..0527da0 --- /dev/null +++ b/bridges/BandcampBridge.php @@ -0,0 +1,63 @@ + array( + 'name' => 'tag', + 'type' => 'text', + 'required' => true + ) + )); + + public function collectData(){ + $html = getSimpleHTMLDOM($this->getURI()) + or returnServerError('No results for this query.'); + + foreach($html->find('li.item') as $release) { + $script = $release->find('div.art', 0)->getAttribute('onclick'); + $uri = ltrim($script, "return 'url("); + $uri = rtrim($uri, "')"); + + $item = array(); + $item['author'] = $release->find('div.itemsubtext', 0)->plaintext + . ' - ' + . $release->find('div.itemtext', 0)->plaintext; + + $item['title'] = $release->find('div.itemsubtext', 0)->plaintext + . ' - ' + . $release->find('div.itemtext', 0)->plaintext; + + $item['content'] = '
' + . $release->find('div.itemsubtext', 0)->plaintext + . ' - ' + . $release->find('div.itemtext', 0)->plaintext; + + $item['id'] = $release->find('a', 0)->getAttribute('href'); + $item['uri'] = $release->find('a', 0)->getAttribute('href'); + $this->items[] = $item; + } + } + + public function getURI(){ + if(!is_null($this->getInput('tag'))) { + return self::URI . 'tag/' . urlencode($this->getInput('tag')) . '?sort_field=date'; + } + + return parent::getURI(); + } + + public function getName(){ + if(!is_null($this->getInput('tag'))) { + return $this->getInput('tag') . ' - Bandcamp Tag'; + } + + return parent::getName(); + } +} diff --git a/bridges/BastaBridge.php b/bridges/BastaBridge.php new file mode 100644 index 0000000..17d3da7 --- /dev/null +++ b/bridges/BastaBridge.php @@ -0,0 +1,34 @@ +find('item') as $element) { + if($limit < 10) { + $item = array(); + $item['title'] = $element->find('title', 0)->innertext; + $item['uri'] = $element->find('guid', 0)->plaintext; + $item['timestamp'] = strtotime($element->find('dc:date', 0)->plaintext); + $item['content'] = replaceImageUrl(getSimpleHTMLDOM($item['uri'])->find('div.texte', 0)->innertext); + $this->items[] = $item; + $limit++; + } + } + } +} diff --git a/bridges/BlaguesDeMerdeBridge.php b/bridges/BlaguesDeMerdeBridge.php new file mode 100644 index 0000000..25c018a --- /dev/null +++ b/bridges/BlaguesDeMerdeBridge.php @@ -0,0 +1,31 @@ +find('article.joke_contener') as $element) { + $item = array(); + $temp = $element->find('a'); + + if(isset($temp[2])) { + $item['content'] = trim($element->find('div.joke_text_contener', 0)->innertext); + $uri = $temp[2]->href; + $item['uri'] = $uri; + $item['title'] = substr($uri, (strrpos($uri, "/") + 1)); + $date = $element->find('li.bdm_date', 0)->innertext; + $time = mktime(0, 0, 0, substr($date, 3, 2), substr($date, 0, 2), substr($date, 6, 4)); + $item['timestamp'] = $time; + $item['author'] = $element->find('li.bdm_pseudo', 0)->innertext; + $this->items[] = $item; + } + } + } +} diff --git a/bridges/BooruprojectBridge.php b/bridges/BooruprojectBridge.php new file mode 100644 index 0000000..6815d37 --- /dev/null +++ b/bridges/BooruprojectBridge.php @@ -0,0 +1,45 @@ + array( + 'p' => array( + 'name' => 'page', + 'type' => 'number' + ), + 't' => array( + 'name' => 'tags' + ) + ), + 'Booru subdomain (subdomain.booru.org)' => array( + 'i' => array( + 'name' => 'Subdomain', + 'required' => true + ) + ) + ); + + const PIDBYPAGE = 20; + + public function getURI(){ + if(!is_null($this->getInput('i'))) { + return 'http://' . $this->getInput('i') . '.booru.org/'; + } + + return parent::getURI(); + } + + public function getName(){ + if(!is_null($this->getInput('i'))) { + return static::NAME . ' ' . $this->getInput('i'); + } + + return parent::getName(); + } +} diff --git a/bridges/CADBridge.php b/bridges/CADBridge.php new file mode 100644 index 0000000..09e3e65 --- /dev/null +++ b/bridges/CADBridge.php @@ -0,0 +1,45 @@ +collectExpandableDatas('http://cdn2.cad-comic.com/rss.xml', 10); + } + + protected function parseItem($newsItem){ + $item = parent::parseItem($newsItem); + $item['content'] = $this->extractCADContent($item['uri']); + return $item; + } + + private function extractCADContent($url) { + $html3 = getSimpleHTMLDOMCached($url); + + // The request might fail due to missing https support or wrong URL + if($html3 == false) + return 'Daily comic not released yet'; + + $htmlpart = explode("/", $url); + + switch ($htmlpart[3]) { + case 'cad': + preg_match_all("/http:\/\/cdn2\.cad-comic\.com\/comics\/cad-\S*png/", $html3, $url2); + break; + case 'sillies': + preg_match_all("/http:\/\/cdn2\.cad-comic\.com\/comics\/sillies-\S*gif/", $html3, $url2); + break; + default: + return 'Daily comic not released yet'; + } + $img = implode($url2[0]); + $html3->clear(); + unset($html3); + if ($img == '') + return 'Daily comic not released yet'; + return ''; + } +} diff --git a/bridges/CNETBridge.php b/bridges/CNETBridge.php new file mode 100644 index 0000000..eefb705 --- /dev/null +++ b/bridges/CNETBridge.php @@ -0,0 +1,93 @@ + You may specify a +topic found in some section URLs, else all topics are selected.'; + + const PARAMETERS = array( array( + 'topic' => array( + 'name' => 'Topic name' + ) + )); + + public function collectData(){ + + function extractFromDelimiters($string, $start, $end){ + if(strpos($string, $start) !== false) { + $section_retrieved = substr($string, strpos($string, $start) + strlen($start)); + $section_retrieved = substr($section_retrieved, 0, strpos($section_retrieved, $end)); + return $section_retrieved; + } + + return false; + } + + function stripWithDelimiters($string, $start, $end){ + while(strpos($string, $start) !== false) { + $section_to_remove = substr($string, strpos($string, $start)); + $section_to_remove = substr($section_to_remove, 0, strpos($section_to_remove, $end) + strlen($end)); + $string = str_replace($section_to_remove, '', $string); + } + + return $string; + } + + function cleanArticle($article_html){ + $article_html = '

' . substr($article_html, strpos($article_html, '

') + 3); + $article_html = stripWithDelimiters($article_html, '', ''); + $article_html = stripWithDelimiters($article_html, ''); + $article_html = stripWithDelimiters($article_html, '

', ''); + return $article_html; + } + + $pageUrl = self::URI . (empty($this->getInput('topic')) ? '' : 'topics/' . $this->getInput('topic') . '/'); + $html = getSimpleHTMLDOM($pageUrl) or returnServerError('Could not request CNET: ' . $pageUrl); + $limit = 0; + + foreach($html->find('div.assetBody') as $element) { + if($limit < 8) { + $article_title = trim($element->find('h2', 0)->plaintext); + $article_uri = self::URI . ($element->find('a', 0)->href); + $article_timestamp = strtotime($element->find('time.assetTime', 0)->plaintext); + $article_author = trim($element->find('a[rel=author]', 0)->plaintext); + + if(!empty($article_title) && !empty($article_uri) && strpos($article_uri, '/news/') !== false) { + $article_html = getSimpleHTMLDOM($article_uri) + or returnServerError('Could not request CNET: ' . $article_uri); + $article_content = trim( + cleanArticle( + extractFromDelimiters( + $article_html, + '
' + . $item['title'] + . '

' + . $this->extractActivityPrice($activity) + . '

'; + + if(isset($zip_filter) + && !(substr($item['title'], 0, strlen($zip_filter)) === $zip_filter)) { + continue; // Skip this item + } + + if(isset($city_filter) + && !(substr($item['title'], strpos($item['title'], ' ') + 1, strlen($city_filter)) === $city_filter)) { + continue; // Skip this item + } + + $this->items[] = $item; + } + } +} diff --git a/bridges/CollegeDeFranceBridge.php b/bridges/CollegeDeFranceBridge.php new file mode 100644 index 0000000..1f81683 --- /dev/null +++ b/bridges/CollegeDeFranceBridge.php @@ -0,0 +1,84 @@ + 'janv.', + '02' => 'févr.', + '03' => 'mars', + '04' => 'avr.', + '05' => 'mai', + '06' => 'juin', + '07' => 'juil.', + '08' => 'août', + '09' => 'sept.', + '10' => 'oct.', + '11' => 'nov.', + '12' => 'déc.' + ); + + // The "API" used by the site returns a list of partial HTML in this form + /*
  • + * + * + * 15 avr. 2016 + * Christopher Hays + * Imagery of Divine Suckling in the Hebrew Bible and the Ancient Near East + * + *
    • + */ + $html = getSimpleHTMLDOM(self::URI + . 'components/search-audiovideo.jsp?fulltext=&siteid=1156951719600&lang=FR&type=all') + or returnServerError('Could not request CollegeDeFrance.'); + + foreach($html->find('a[data-target]') as $element) { + $item = array(); + $item['title'] = $element->find('.title', 0)->plaintext; + + // Most relative URLs contains an hour in addition to the date, so let's use it + // + // + // Sometimes there's an __1, perhaps it signifies an update + // "/site/patrick-boucheron/seminar-2016-05-03-18h00__1.htm" + // + // But unfortunately some don't have any hours info + // + $timezone = new DateTimeZone('Europe/Paris'); + + // strpos($element->href, '201') will break in 2020 but it'll + // probably break prior to then due to site changes anyway + $d = DateTime::createFromFormat( + '!Y-m-d-H\hi', + substr($element->href, strpos($element->href, '201'), 16), + $timezone + ); + + if(!$d) { + $d = DateTime::createFromFormat( + '!d m Y', + trim(str_replace( + array_values($months), + array_keys($months), + $element->find('.date', 0)->plaintext + )), + $timezone + ); + } + + $item['timestamp'] = $d->format('U'); + $item['content'] = $element->find('.lecturer', 0)->innertext + . ' - ' + . $element->find('.title', 0)->innertext; + + $item['uri'] = self::URI . $element->href; + $this->items[] = $item; + } + } +} diff --git a/bridges/CommonDreamsBridge.php b/bridges/CommonDreamsBridge.php new file mode 100644 index 0000000..e4dcb63 --- /dev/null +++ b/bridges/CommonDreamsBridge.php @@ -0,0 +1,26 @@ +collectExpandableDatas('http://www.commondreams.org/rss.xml', 10); + } + + protected function parseItem($newsItem){ + $item = parent::parseItem($newsItem); + $item['content'] = $this->extractContent($item['uri']); + return $item; + } + + private function extractContent($url){ + $html3 = getSimpleHTMLDOMCached($url); + $text = $html3->find('div[class=field--type-text-with-summary]', 0)->innertext; + $html3->clear(); + unset ($html3); + return $text; + } +} diff --git a/bridges/CopieDoubleBridge.php b/bridges/CopieDoubleBridge.php new file mode 100644 index 0000000..767cdce --- /dev/null +++ b/bridges/CopieDoubleBridge.php @@ -0,0 +1,35 @@ +find('table table', 2); + + foreach($table->find('tr') as $element) { + $td = $element->find('td', 0); + + if($td->class === 'couleur_1') { + $item = array(); + $title = $td->innertext; + $pos = strpos($title, 'innertext, '/images/suivant.gif') === false) { + $a = $element->find('a', 0); + $item['uri'] = self::URI . $a->href; + $content = str_replace('src="/', 'src="/' . self::URI, $element->find("td", 0)->innertext); + $content = str_replace('href="/', 'href="' . self::URI, $content); + $item['content'] = $content; + $this->items[] = $item; + } + } + } +} diff --git a/bridges/CourrierInternationalBridge.php b/bridges/CourrierInternationalBridge.php new file mode 100644 index 0000000..1573863 --- /dev/null +++ b/bridges/CourrierInternationalBridge.php @@ -0,0 +1,55 @@ +find("article"); + $article_count = 1; + + foreach($element as $article) { + $item = array(); + + $item['uri'] = $article->parent->getAttribute('href'); + + if(strpos($item['uri'], 'http') === false) { + $item['uri'] = self::URI . $item['uri']; + } + + $page = getSimpleHTMLDOMCached($item['uri']); + + $content = $page->find('.article-text', 0); + + if(!$content) { + $content = $page->find('.depeche-text', 0); + } + + $item['content'] = sanitize($content); + $item['title'] = strip_tags($article->find('.title', 0)); + + $dateTime = date_parse($page->find('time', 0)); + + $item['timestamp'] = mktime( + $dateTime['hour'], + $dateTime['minute'], + $dateTime['second'], + $dateTime['month'], + $dateTime['day'], + $dateTime['year'] + ); + + $this->items[] = $item; + $article_count ++; + + if($article_count > 5) + break; + } + } +} diff --git a/bridges/CpasbienBridge.php b/bridges/CpasbienBridge.php new file mode 100644 index 0000000..19efd84 --- /dev/null +++ b/bridges/CpasbienBridge.php @@ -0,0 +1,74 @@ + array( + 'name' => 'Search', + 'required' => true, + 'title' => 'Type your search' + ) + )); + + public function collectData(){ + $request = str_replace(" ", "-", trim($this->getInput('q'))); + $html = getSimpleHTMLDOM(self::URI . '/recherche/' . urlencode($request) . '.html') + or returnServerError('No results for this query.'); + + foreach($html->find('#gauche', 0)->find('div') as $episode) { + if($episode->getAttribute('class') == 'ligne0' + || $episode->getAttribute('class') == 'ligne1') { + + $urlepisode = $episode->find('a', 0)->getAttribute('href'); + $htmlepisode = getSimpleHTMLDOMCached($urlepisode, 86400 * 366 * 30); + + $item = array(); + $item['author'] = $episode->find('a', 0)->text(); + $item['title'] = $episode->find('a', 0)->text(); + $item['pubdate'] = $this->getCachedDate($urlepisode); + $textefiche = $htmlepisode->find('#textefiche', 0)->find('p', 1); + + if(isset($textefiche)) { + $item['content'] = $textefiche->text(); + } else { + $p = $htmlepisode->find('#textefiche', 0)->find('p'); + if(!empty($p)) { + $item['content'] = $htmlepisode->find('#textefiche', 0)->find('p', 0)->text(); + } + } + + $item['id'] = $episode->find('a', 0)->getAttribute('href'); + $item['uri'] = self::URI . $htmlepisode->find('#telecharger', 0)->getAttribute('href'); + $this->items[] = $item; + } + } + } + + public function getName(){ + if(!is_null($this->getInput('q'))) { + return $this->getInput('q') . ' : ' . self::NAME; + } + + return parent::getName(); + } + + private function getCachedDate($url){ + debugMessage('getting pubdate from url ' . $url . ''); + + // Initialize cache + $cache = Cache::create('FileCache'); + $cache->setPath(CACHE_DIR . '/pages'); + + $params = [$url]; + $cache->setParameters($params); + + // Get cachefile timestamp + $time = $cache->getTime(); + return ($time !== false ? $time : time()); + } +} diff --git a/bridges/CryptomeBridge.php b/bridges/CryptomeBridge.php new file mode 100644 index 0000000..8a3936f --- /dev/null +++ b/bridges/CryptomeBridge.php @@ -0,0 +1,45 @@ + array( + 'name' => 'number of elements', + 'type' => 'number', + 'defaultValue' => 20, + 'exampleValue' => 10 + ) + )); + + public function collectData(){ + $html = getSimpleHTMLDOM(self::URI) + or returnServerError('Could not request Cryptome.'); + + $number = $this->getInput('n'); + + /* number of documents */ + if(!empty($number)) { + $num = min($number, 20); + } + + foreach($html->find('pre') as $element) { + for($i = 0; $i < $num; ++$i) { + $item = array(); + $item['uri'] = self::URI . substr($element->find('a', $i)->href, 20); + $item['title'] = substr($element->find('b', $i)->plaintext, 22); + $item['content'] = preg_replace( + '#http://cryptome.org/#', + self::URI, + $element->find('b', $i)->innertext + ); + $this->items[] = $item; + } + break; + } + } +} diff --git a/bridges/DailymotionBridge.php b/bridges/DailymotionBridge.php new file mode 100644 index 0000000..d075041 --- /dev/null +++ b/bridges/DailymotionBridge.php @@ -0,0 +1,123 @@ + array( + 'u' => array( + 'name' => 'username', + 'required' => true + ) + ), + 'By playlist id' => array( + 'p' => array( + 'name' => 'playlist id', + 'required' => true + ) + ), + 'From search results' => array( + 's' => array( + 'name' => 'Search keyword', + 'required' => true + ), + 'pa' => array( + 'name' => 'Page', + 'type' => 'number' + ) + ) + ); + + protected function getMetadata($id){ + $metadata = array(); + $html2 = getSimpleHTMLDOM(self::URI . 'video/' . $id); + if(!$html2) { + return $metadata; + } + + $metadata['title'] = $html2->find('meta[property=og:title]', 0)->getAttribute('content'); + $metadata['timestamp'] = strtotime( + $html2->find('meta[property=video:release_date]', 0)->getAttribute('content') + ); + $metadata['thumbnailUri'] = $html2->find('meta[property=og:image]', 0)->getAttribute('content'); + $metadata['uri'] = $html2->find('meta[property=og:url]', 0)->getAttribute('content'); + return $metadata; + } + + public function collectData(){ + $html = ''; + $limit = 5; + $count = 0; + + $html = getSimpleHTMLDOM($this->getURI()) + or returnServerError('Could not request Dailymotion.'); + + foreach($html->find('div.media a.preview_link') as $element) { + if($count < $limit) { + $item = array(); + $item['id'] = str_replace('/video/', '', strtok($element->href, '_')); + $metadata = $this->getMetadata($item['id']); + if(empty($metadata)) { + continue; + } + $item['uri'] = $metadata['uri']; + $item['title'] = $metadata['title']; + $item['timestamp'] = $metadata['timestamp']; + + $item['content'] = '
    ' + . $item['title'] + . ''; + + $this->items[] = $item; + $count++; + } + } + } + + public function getName(){ + switch($this->queriedContext) { + case 'By username': + $specific = $this->getInput('u'); + break; + case 'By playlist id': + $specific = strtok($this->getInput('p'), '_'); + break; + case 'From search results': + $specific = $this->getInput('s'); + break; + default: return parent::getName(); + } + + return $specific . ' : Dailymotion Bridge'; + } + + public function getURI(){ + $uri = self::URI; + switch($this->queriedContext) { + case 'By username': + $uri .= 'user/' . urlencode($this->getInput('u')) . '/1'; + break; + case 'By playlist id': + $uri .= 'playlist/' . urlencode(strtok($this->getInput('p'), '_')); + break; + case 'From search results': + $uri .= 'search/' . urlencode($this->getInput('s')); + if($this->getInput('pa')) { + $uri .= '/' . $this->getInput('pa'); + } + break; + default: return parent::getURI(); + } + return $uri; + } +} diff --git a/bridges/DanbooruBridge.php b/bridges/DanbooruBridge.php new file mode 100644 index 0000000..f2cddf4 --- /dev/null +++ b/bridges/DanbooruBridge.php @@ -0,0 +1,62 @@ + array( + 'p' => array( + 'name' => 'page', + 'defaultValue' => 1, + 'type' => 'number' + ), + 't' => array( + 'name' => 'tags' + ) + ), + 0 => array() + ); + + const PATHTODATA = 'article'; + const IDATTRIBUTE = 'data-id'; + + protected function getFullURI(){ + return $this->getURI() + . 'posts?&page=' . $this->getInput('p') + . '&tags=' . urlencode($this->getInput('t')); + } + + protected function getItemFromElement($element){ + // Fix links + defaultLinkTo($element, $this->getURI()); + + $item = array(); + $item['uri'] = $element->find('a', 0)->href; + $item['postid'] = (int)preg_replace("/[^0-9]/", '', $element->getAttribute(static::IDATTRIBUTE)); + $item['timestamp'] = time(); + $thumbnailUri = $element->find('img', 0)->src; + $item['tags'] = $element->find('img', 0)->getAttribute('alt'); + $item['title'] = $this->getName() . ' | ' . $item['postid']; + $item['content'] = '
    Tags: ' + . $item['tags']; + + return $item; + } + + public function collectData(){ + $html = getSimpleHTMLDOM($this->getFullURI()) + or returnServerError('Could not request ' . $this->getName()); + + foreach($html->find(static::PATHTODATA) as $element) { + $this->items[] = $this->getItemFromElement($element); + } + } +} diff --git a/bridges/DansTonChatBridge.php b/bridges/DansTonChatBridge.php new file mode 100644 index 0000000..545f162 --- /dev/null +++ b/bridges/DansTonChatBridge.php @@ -0,0 +1,23 @@ +find('div.item') as $element) { + $item = array(); + $item['uri'] = $element->find('a', 0)->href; + $item['title'] = 'DansTonChat ' . $element->find('a', 1)->plaintext; + $item['content'] = $element->find('a', 0)->innertext; + $this->items[] = $item; + } + } +} diff --git a/bridges/DauphineLibereBridge.php b/bridges/DauphineLibereBridge.php new file mode 100644 index 0000000..7547d74 --- /dev/null +++ b/bridges/DauphineLibereBridge.php @@ -0,0 +1,56 @@ + array( + 'name' => 'Catégorie de l\'article', + 'type' => 'list', + 'values' => array( + 'À la une' => '', + 'France Monde' => 'france-monde', + 'Faits Divers' => 'faits-divers', + 'Économie et Finance' => 'economie-et-finance', + 'Politique' => 'politique', + 'Sport' => 'sport', + 'Ain' => 'ain', + 'Alpes-de-Haute-Provence' => 'haute-provence', + 'Hautes-Alpes' => 'hautes-alpes', + 'Ardèche' => 'ardeche', + 'Drôme' => 'drome', + 'Isère Sud' => 'isere-sud', + 'Savoie' => 'savoie', + 'Haute-Savoie' => 'haute-savoie', + 'Vaucluse' => 'vaucluse' + ) + ) + )); + + public function collectData(){ + $url = self::URI . 'rss'; + + if(empty($this->getInput('u'))) { + $url = self::URI . $this->getInput('u') . '/rss'; + } + + $this->collectExpandableDatas($url, 10); + } + + protected function parseItem($newsItem){ + $item = parent::parseItem($newsItem); + $item['content'] = $this->extractContent($item['uri']); + return $item; + } + + private function extractContent($url){ + $html2 = getSimpleHTMLDOMCached($url); + $text = $html2->find('div.column', 0)->innertext; + $text = preg_replace('@]*?>.*?@si', '', $text); + return $text; + } +} diff --git a/bridges/DemoBridge.php b/bridges/DemoBridge.php new file mode 100644 index 0000000..ea2088e --- /dev/null +++ b/bridges/DemoBridge.php @@ -0,0 +1,46 @@ + array( + 'testCheckbox' => array( + 'type' => 'checkbox', + 'name' => 'test des checkbox' + ) + ), + 'testList' => array( + 'testList' => array( + 'type' => 'list', + 'name' => 'test des listes', + 'values' => array( + 'Test' => 'test', + 'Test 2' => 'test2' + ) + ) + ), + 'testNumber' => array( + 'testNumber' => array( + 'type' => 'number', + 'name' => 'test des numéros', + 'exampleValue' => '1515632' + ) + ) + ); + + public function collectData(){ + + $item = array(); + $item['author'] = "Me!"; + $item['title'] = "Test"; + $item['content'] = "Awesome content !"; + $item['id'] = "Lalala"; + $item['uri'] = "http://example.com/test"; + + $this->items[] = $item; + } +} diff --git a/bridges/DeveloppezDotComBridge.php b/bridges/DeveloppezDotComBridge.php new file mode 100644 index 0000000..5719cf3 --- /dev/null +++ b/bridges/DeveloppezDotComBridge.php @@ -0,0 +1,47 @@ +collectExpandableDatas(self::URI . 'index/rss', 15); + } + + protected function parseItem($newsItem){ + $item = parent::parseItem($newsItem); + $item['content'] = $this->extractContent($item['uri']); + return $item; + } + + // F***ing quotes from Microsoft Word badly encoded, here was the trick: + // http://stackoverflow.com/questions/1262038/how-to-replace-microsoft-encoded-quotes-in-php + private function convertSmartQuotes($string) + { + $search = array(chr(145), + chr(146), + chr(147), + chr(148), + chr(151)); + + $replace = array( + "'", + "'", + '"', + '"', + '-' + ); + + return str_replace($search, $replace, $string); + } + + private function extractContent($url){ + $articleHTMLContent = getSimpleHTMLDOMCached($url); + $text = $this->convertSmartQuotes($articleHTMLContent->find('div.content', 0)->innertext); + $text = utf8_encode($text); + return trim($text); + } +} diff --git a/bridges/DiceBridge.php b/bridges/DiceBridge.php new file mode 100644 index 0000000..dc6ea15 --- /dev/null +++ b/bridges/DiceBridge.php @@ -0,0 +1,120 @@ + array( + 'name' => 'With at least one of the words', + 'required' => false, + ), + 'for_all' => array( + 'name' => 'With all of the words', + 'required' => false, + ), + 'for_exact' => array( + 'name' => 'With the exact phrase', + 'required' => false, + ), + 'for_none' => array( + 'name' => 'With none of these words', + 'required' => false, + ), + 'for_jt' => array( + 'name' => 'Within job title', + 'required' => false, + ), + 'for_com' => array( + 'name' => 'Within company name', + 'required' => false, + ), + 'for_loc' => array( + 'name' => 'City, State, or ZIP code', + 'required' => false, + ), + 'radius' => array( + 'name' => 'Radius in miles', + 'type' => 'list', + 'required' => false, + 'values' => array( + 'Exact Location' => 'El', + 'Within 5 miles' => '5', + 'Within 10 miles' => '10', + 'Within 20 miles' => '20', + 'Within 30 miles' => '0', + 'Within 40 miles' => '40', + 'Within 50 miles' => '50', + 'Within 75 miles' => '75', + 'Within 100 miles' => '100', + ), + 'defaultValue' => '0', + ), + 'jtype' => array( + 'name' => 'Job type', + 'type' => 'list', + 'required' => false, + 'values' => array( + 'Full-Time' => 'Full Time', + 'Part-Time' => 'Part Time', + 'Contract - Independent' => 'Contract Independent', + 'Contract - W2' => 'Contract W2', + 'Contract to Hire - Independent' => 'C2H Independent', + 'Contract to Hire - W2' => 'C2H W2', + 'Third Party - Contract - Corp-to-Corp' => 'Contract Corp-To-Corp', + 'Third Party - Contract to Hire - Corp-to-Corp' => 'C2H Corp-To-Corp', + ), + 'defaultValue' => 'Full Time', + ), + 'telecommute' => array( + 'name' => 'Telecommute', + 'type' => 'checkbox', + ), + )); + + public function collectData() { + $uri = 'https://www.dice.com/jobs/advancedResult.html'; + $uri .= '?for_one=' . urlencode($this->getInput('for_one')); + $uri .= '&for_all=' . urlencode($this->getInput('for_all')); + $uri .= '&for_exact=' . urlencode($this->getInput('for_exact')); + $uri .= '&for_none=' . urlencode($this->getInput('for_none')); + $uri .= '&for_jt=' . urlencode($this->getInput('for_jt')); + $uri .= '&for_com=' . urlencode($this->getInput('for_com')); + $uri .= '&for_loc=' . urlencode($this->getInput('for_loc')); + if ($this->getInput('jtype')) { + $uri .= '&jtype=' . urlencode($this->getInput('jtype')); + } + $uri .= '&sort=date&limit=100'; + $uri .= '&radius=' . urlencode($this->getInput('radius')); + if ($this->getInput('telecommute')) { + $uri .= '&telecommute=true'; + } + + $html = getSimpleHTMLDOM($uri) + or returnServerError('Could not request Dice.'); + foreach($html->find('div.complete-serp-result-div') as $element) { + $item = array(); + // Title + $masterLink = $element->find('a[id^=position]', 0); + $item['title'] = $masterLink->title; + // URL + $uri = $masterLink->href; + // $uri = substr($uri, 0, strrpos($uri, '?')); + $item['uri'] = substr($uri, 0, strrpos($uri, '?')); + // ID + $item['id'] = $masterLink->value; + // Image + $image = $element->find('img', 0); + if ($image) + $item['image'] = $image->getAttribute('src'); + // Content + $shortdesc = $element->find('.shortdesc', '0'); + $shortdesc = ($shortdesc) ? $shortdesc->innertext : ''; + $item['content'] = $shortdesc; + $this->items[] = $item; + } + } +} diff --git a/bridges/DilbertBridge.php b/bridges/DilbertBridge.php new file mode 100644 index 0000000..959a91a --- /dev/null +++ b/bridges/DilbertBridge.php @@ -0,0 +1,36 @@ +getURI()) + or returnServerError('Could not request Dilbert: ' . $this->getURI()); + + foreach($html->find('section.comic-item') as $element) { + + $img = $element->find('img', 0); + $link = $element->find('a', 0); + $comic = $img->src; + $title = $link->alt; + $url = $link->href; + $date = substr($url, 25); + if (empty($title)) + $title = 'Dilbert Comic Strip on ' . $date; + $date = strtotime($date); + + $item = array(); + $item['uri'] = $url; + $item['title'] = $title; + $item['author'] = 'Scott Adams'; + $item['timestamp'] = $date; + $item['content'] = '' . $img->alt . ''; + $this->items[] = $item; + } + } +} diff --git a/bridges/DollbooruBridge.php b/bridges/DollbooruBridge.php new file mode 100644 index 0000000..5ed4119 --- /dev/null +++ b/bridges/DollbooruBridge.php @@ -0,0 +1,9 @@ + array( + 'name' => 'keyword', + 'required' => true + ), + 'sort' => array( + 'name' => 'sort by', + 'type' => 'list', + 'required' => false, + 'values' => array( + 'date' => self::SORT_DATE, + 'relevance' => self::SORT_RELEVANCE + ), + 'defaultValue' => self::SORT_DATE + ) + )); + + public function collectData(){ + $html = getSimpleHTMLDOM(self::URI . 'html/?kd=-1&q=' . $this->getInput('u') . $this->getInput('sort')) + or returnServerError('Could not request DuckDuckGo.'); + + foreach($html->find('div.results_links') as $element) { + $item = array(); + $item['uri'] = $element->find('a', 0)->href; + $item['title'] = $element->find('a', 1)->innertext; + $item['content'] = $element->find('div.snippet', 0)->plaintext; + $this->items[] = $item; + } + } +} diff --git a/bridges/EZTVBridge.php b/bridges/EZTVBridge.php new file mode 100644 index 0000000..4fb9e57 --- /dev/null +++ b/bridges/EZTVBridge.php @@ -0,0 +1,67 @@ + array( + 'name' => 'Show ids', + 'exampleValue' => 'showID1,showID2,…', + 'required' => true + ) + )); + + public function collectData(){ + + // Make timestamp from relative released time in table + function makeTimestamp($relativeReleaseTime){ + + $relativeDays = 0; + $relativeHours = 0; + + foreach(explode(" ", $relativeReleaseTime) as $relativeTimeElement) { + if(substr($relativeTimeElement, -1) == "d") $relativeDays = substr($relativeTimeElement, 0, -1); + if(substr($relativeTimeElement, -1) == "h") $relativeHours = substr($relativeTimeElement, 0, -1); + } + return mktime(date('h') - $relativeHours, 0, 0, date('m'), date('d') - $relativeDays, date('Y')); + } + + // Loop on show ids + $showList = explode(",", $this->getInput('i')); + foreach($showList as $showID) { + + // Get show page + $html = getSimpleHTMLDOM(self::URI . 'shows/' . rawurlencode($showID) . '/') + or returnServerError('Could not request EZTV for id "' . $showID . '"'); + + // Loop on each element that look like an episode entry... + foreach($html->find('.forum_header_border') as $element) { + + // Filter entries that are not episode entries + $ep = $element->find('td', 1); + if(empty($ep)) continue; + $epinfo = $ep->find('.epinfo', 0); + $released = $element->find('td', 3); + if(empty($epinfo)) continue; + if(empty($released->plaintext)) continue; + + // Filter entries that are older than 1 week + if($released->plaintext == '>1 week') continue; + + // Fill item + $item = array(); + $item['uri'] = self::URI . $epinfo->href; + $item['id'] = $item['uri']; + $item['timestamp'] = makeTimestamp($released->plaintext); + $item['title'] = $epinfo->plaintext; + $item['content'] = $epinfo->alt; + if(isset($item['title'])) + $this->items[] = $item; + } + } + } +} diff --git a/bridges/EliteDangerousGalnetBridge.php b/bridges/EliteDangerousGalnetBridge.php new file mode 100644 index 0000000..86a1bbf --- /dev/null +++ b/bridges/EliteDangerousGalnetBridge.php @@ -0,0 +1,35 @@ +find('div.article') as $element) { + $item = array(); + + $uri = $element->find('h3 a', 0)->href; + $uri = self::URI . substr($uri, strlen('/galnet/')); + $item['uri'] = $uri; + + $title = $element->find('h3 a', 0)->plaintext; + $item['title'] = substr($title, 1); //remove the space between icon and title + + $content = $element->find('p', -1)->innertext; + $item['content'] = $content; + + $date = $element->find('p.small', 0)->innertext; + $article_year = substr($date, -4) - 1286; //Convert E:D date to actual date + $date = substr($date, 0, -4) . $article_year; + $item['timestamp'] = strtotime($date); + + $this->items[] = $item; + } + } +} diff --git a/bridges/ElsevierBridge.php b/bridges/ElsevierBridge.php new file mode 100644 index 0000000..f6ba7dd --- /dev/null +++ b/bridges/ElsevierBridge.php @@ -0,0 +1,75 @@ + array( + 'name' => 'Journal name', + 'required' => true, + 'exampleValue' => 'academic-pediactrics', + 'title' => 'Insert html-part of your journal' + ) + )); + + // Extracts the list of names from an article as string + private function extractArticleName($article){ + $names = $article->find('small', 0); + if($names) + return trim($names->plaintext); + return ''; + } + + // Extracts the timestamp from an article + private function extractArticleTimestamp($article){ + $time = $article->find('.article-info', 0); + if($time) { + $timestring = trim($time->plaintext); + /* + The format depends on the age of an article: + - Available online 29 July 2016 + - July 2016 + - May–June 2016 + */ + if(preg_match('/\S*(\d+\s\S+\s\d{4})/ims', $timestring, $matches)) { + return strtotime($matches[0]); + } elseif (preg_match('/[A-Za-z]+\-([A-Za-z]+\s\d{4})/ims', $timestring, $matches)) { + return strtotime($matches[0]); + } elseif (preg_match('/([A-Za-z]+\s\d{4})/ims', $timestring, $matches)) { + return strtotime($matches[0]); + } else { + return 0; + } + } + return 0; + } + + // Extracts the content from an article + private function extractArticleContent($article){ + $content = $article->find('.article-content', 0); + if($content) { + return trim($content->plaintext); + } + return ''; + } + + public function collectData(){ + $uri = self::URI . $this->getInput('j') . '/recent-articles/'; + $html = getSimpleHTMLDOM($uri) + or returnServerError('No results for Elsevier journal ' . $this->getInput('j')); + + foreach($html->find('.pod-listing') as $article) { + $item = array(); + $item['uri'] = $article->find('.pod-listing-header>a', 0)->getAttribute('href') . '?np=y'; + $item['title'] = $article->find('.pod-listing-header>a', 0)->plaintext; + $item['author'] = $this->extractArticleName($article); + $item['timestamp'] = $this->extractArticleTimestamp($article); + $item['content'] = $this->extractArticleContent($article); + $this->items[] = $item; + } + } +} diff --git a/bridges/EstCeQuonMetEnProdBridge.php b/bridges/EstCeQuonMetEnProdBridge.php new file mode 100644 index 0000000..db9d1d5 --- /dev/null +++ b/bridges/EstCeQuonMetEnProdBridge.php @@ -0,0 +1,37 @@ +getURI()) + or returnServerError('Could not request EstCeQuonMetEnProd: ' . $this->getURI()); + + $item = array(); + $item['uri'] = $this->getURI() . '#' . date('Y-m-d'); + $item['title'] = $this->getName(); + $item['author'] = 'Nicolas Hoffmann'; + $item['timestamp'] = strtotime('today midnight'); + $item['content'] = str_replace( + 'src="/', + 'src="' . $this->getURI(), + trim(extractFromDelimiters($html->outertext, '', '

    ')) + ); + + $this->items[] = $item; + } +} diff --git a/bridges/EtsyBridge.php b/bridges/EtsyBridge.php new file mode 100644 index 0000000..311d910 --- /dev/null +++ b/bridges/EtsyBridge.php @@ -0,0 +1,83 @@ + array( + 'name' => 'Search query', + 'type' => 'text', + 'required' => true, + 'title' => 'Insert your search term here', + 'exampleValue' => 'Enter your search term' + ), + 'queryextension' => array( + 'name' => 'Query extension', + 'type' => 'text', + 'requied' => false, + 'title' => 'Insert additional query parts here +(anything after ?search=)', + 'exampleValue' => '&explicit=1&locationQuery=2921044' + ), + 'showimage' => array( + 'name' => 'Show image in content', + 'type' => 'checkbox', + 'requrired' => false, + 'title' => 'Activate to show the image in the content', + 'defaultValue' => false + ) + ) + ); + + public function collectData(){ + $html = getSimpleHTMLDOM($this->getURI()) + or returnServerError('Failed to receive ' . $this->getURI()); + + $results = $html->find('div.block-grid-item'); + + foreach($results as $result) { + // Skip banner cards (ads for categories) + if($result->find('a.banner-card')) + continue; + + $item = array(); + + $item['title'] = $result->find('a', 0)->title; + $item['uri'] = $result->find('a', 0)->href; + $item['author'] = $result->find('div.card-shop-name', 0)->plaintext; + + $item['content'] = '

    ' + . $result->find('div.card-price', 0)->plaintext + . '

    ' + . $result->find('div.card-title', 0)->plaintext + . '

    '; + + $image = $result->find('img.placeholder', 0)->src; + + if($this->getInput('showimage')) { + $item['content'] .= ''; + } + + $item['enclosures'] = array($image); + + $this->items[] = $item; + } + } + + public function getURI(){ + if(!is_null($this->getInput('query'))) { + $uri = self::URI . '/search?q=' . urlencode($this->getInput('query')); + + if(!is_null($this->getInput('queryextension'))) { + $uri .= $this->getInput('queryextension'); + } + + return $uri; + } + + return parent::getURI(); + } +} diff --git a/bridges/FB2Bridge.php b/bridges/FB2Bridge.php new file mode 100644 index 0000000..7d78b87 --- /dev/null +++ b/bridges/FB2Bridge.php @@ -0,0 +1,281 @@ + array( + 'name' => 'Username', + 'required' => true + ) + )); + + public function collectData(){ + + function extractFromDelimiters($string, $start, $end){ + if(strpos($string, $start) !== false) { + $section_retrieved = substr($string, strpos($string, $start) + strlen($start)); + $section_retrieved = substr($section_retrieved, 0, strpos($section_retrieved, $end)); + return $section_retrieved; + } + + return false; + } + + //Utility function for cleaning a Facebook link + $unescape_fb_link = function($matches){ + if(is_array($matches) && count($matches) > 1) { + $link = $matches[1]; + if(strpos($link, '/') === 0) + $link = self::URI . $link . '"'; + if(strpos($link, 'facebook.com/l.php?u=') !== false) + $link = urldecode(extractFromDelimiters($link, 'facebook.com/l.php?u=', '&')); + return ' href="' . $link . '"'; + } + }; + + //Utility function for converting facebook emoticons + $unescape_fb_emote = function($matches){ + static $facebook_emoticons = array( + 'smile' => ':)', + 'frown' => ':(', + 'tongue' => ':P', + 'grin' => ':D', + 'gasp' => ':O', + 'wink' => ';)', + 'pacman' => ':<', + 'grumpy' => '>_<', + 'unsure' => ':/', + 'cry' => ':\'(', + 'kiki' => '^_^', + 'glasses' => '8-)', + 'sunglasses' => 'B-)', + 'heart' => '<3', + 'devil' => ']:D', + 'angel' => '0:)', + 'squint' => '-_-', + 'confused' => 'o_O', + 'upset' => 'xD', + 'colonthree' => ':3', + 'like' => '👍'); + $len = count($matches); + if ($len > 1) + for ($i = 1; $i < $len; $i++) + foreach ($facebook_emoticons as $name => $emote) + if ($matches[$i] === $name) + return $emote; + return $matches[0]; + }; + + if($this->getInput('u') !== null) { + $page = 'https://touch.facebook.com/' . $this->getInput('u'); + $cookies = $this->getCookies($page); + $pageID = $this->getPageID($page, $cookies); + + if($pageID === null) { + echo <<buildContent($fileContent); + $author = $this->getInput('u'); + + foreach($html->find("article") as $content) { + + $item = array(); + + $item['uri'] = "http://touch.facebook.com" + . $content->find("div[class='_52jc _5qc4 _24u0 _36xo']", 0)->find("a", 0)->getAttribute("href"); + + if($content->find("header", 0) !== null) { + $content->find("header", 0)->innertext = ""; + } + + if($content->find("footer", 0) !== null) { + $content->find("footer", 0)->innertext = ""; + } + + //Remove html nodes, keep only img, links, basic formatting + $content = strip_tags($content, '

    '); + + //Adapt link hrefs: convert relative links into absolute links and bypass external link redirection + $content = preg_replace_callback('/ href=\"([^"]+)\"/i', $unescape_fb_link, $content); + + //Clean useless html tag properties and fix link closing tags + foreach (array( + 'onmouseover', + 'onclick', + 'target', + 'ajaxify', + 'tabindex', + 'class', + 'style', + 'data-[^=]*', + 'aria-[^=]*', + 'role', + 'rel', + 'id') as $property_name) + $content = preg_replace('/ ' . $property_name . '=\"[^"]*\"/i', '', $content); + $content = preg_replace('/<\/a [^>]+>/i', '', $content); + + //Convert textual representation of emoticons eg + // "smile emoticon" back to ASCII emoticons eg ":)" + $content = preg_replace_callback('/([^ <>]+) ([^<>]+)<\/u><\/i>/i', $unescape_fb_emote, $content); + + $item['content'] = $content; + + $title = $author; + if (strlen($title) > 24) + $title = substr($title, 0, strpos(wordwrap($title, 24), "\n")) . '...'; + $title = $title . ' | ' . strip_tags($content); + if (strlen($title) > 64) + $title = substr($title, 0, strpos(wordwrap($title, 64), "\n")) . '...'; + + $item['title'] = $title; + $item['author'] = $author; + + array_push($this->items, $item); + } + } + + + // Currently not used. Is used to get more than only 3 elements, as they appear on another page. + private function computeNextLink($string, $pageID){ + + $regex = implode( + '', + array( + "/timeline_unit", + "\\\\\\\\u00253A1", + "\\\\\\\\u00253A([0-9]*)", + "\\\\\\\\u00253A([0-9]*)", + "\\\\\\\\u00253A([0-9]*)", + "\\\\\\\\u00253A([0-9]*)/" + ) + ); + + preg_match($regex, $string, $result); + + return implode( + '', + array( + "https://touch.facebook.com/pages_reaction_units/more/?page_id=", + $pageID, + "&cursor=%7B%22timeline_cursor%22%3A%22timeline_unit%3A1%3A", + $result[1], + "%3A", + $result[2], + "%3A", + $result[3], + "%3A", + $result[4], + "%22%2C%22timeline_section_cursor%22%3A%7B%7D%2C%22", + "has_next_page%22%3Atrue%7D&surface=mobile_page_home&unit_count=3" + ) + ); + } + + //Builds the HTML from the encoded JS that Facebook provides. + private function buildContent($pageContent){ + + $regex = "/\\\"html\\\":\\\"(.*?)\\\",\\\"replace/"; + preg_match($regex, $pageContent, $result); + + return str_get_html(html_entity_decode(json_decode('"' . $result[1] . '"'))); + } + + + //Builds the cookie from the page, as Facebook sometimes refuses to give + //the page if no cookie is provided. + private function getCookies($pageURL){ + + $ctx = stream_context_create(array( + 'http' => array( + 'user_agent' => "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:46.0) Gecko/20100101 Firefox/46.0", + 'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8' + ) + ) + ); + $a = file_get_contents($pageURL, 0, $ctx); + + //First request to get the cookie + $cookies = ""; + foreach($http_response_header as $hdr) { + if(strpos($hdr, "Set-Cookie") !== false) { + $cLine = explode(":", $hdr)[1]; + $cLine = explode(";", $cLine)[0]; + $cookies .= ";" . $cLine; + } + } + + return substr($cookies, 1); + } + + //Get the page ID from the Facebook page. + private function getPageID($page, $cookies){ + + $context = stream_context_create(array( + 'http' => array( + 'user_agent' => "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:46.0) Gecko/20100101 Firefox/46.0", + 'header' => 'Cookie: ' . $cookies + ) + ) + ); + + $pageContent = file_get_contents($page, 0, $context); + + if(strpos($pageContent, "signup-button") != false) { + return -1; + } + + //Get the page ID if we don't have a captcha + $regex = "/page_id=([0-9]*)&/"; + preg_match($regex, $pageContent, $matches); + + if(count($matches) > 0) { + return $matches[1]; + } + + //Get the page ID if we do have a captcha + $regex = "/\"pageID\":\"([0-9]*)\"/"; + preg_match($regex, $pageContent, $matches); + + return $matches[1]; + + } + + public function getName(){ + return (isset($this->name) ? $this->name . ' - ' : '') . 'Facebook Bridge'; + } + + public function getURI(){ + return 'http://facebook.com'; + } + + public function getCacheDuration(){ + return 60 * 60 * 3; // 5 minutes + } +} diff --git a/bridges/FacebookBridge.php b/bridges/FacebookBridge.php new file mode 100644 index 0000000..cc16196 --- /dev/null +++ b/bridges/FacebookBridge.php @@ -0,0 +1,305 @@ + array( + 'name' => 'Username', + 'required' => true + ), + 'media_type' => array( + 'name' => 'Media type', + 'type' => 'list', + 'required' => false, + 'values' => array( + 'All' => 'all', + 'Video' => 'video', + 'No Video' => 'novideo' + ), + 'defaultValue' => 'all' + ) + )); + + private $authorName = ''; + + public function collectData(){ + + //Extract a string using start and end delimiters + function extractFromDelimiters($string, $start, $end){ + if(strpos($string, $start) !== false) { + $section_retrieved = substr($string, strpos($string, $start) + strlen($start)); + $section_retrieved = substr($section_retrieved, 0, strpos($section_retrieved, $end)); + return $section_retrieved; + } + + return false; + } + + //Utility function for cleaning a Facebook link + $unescape_fb_link = function($matches){ + if(is_array($matches) && count($matches) > 1) { + $link = $matches[1]; + if(strpos($link, '/') === 0) + $link = self::URI . $link . '"'; + if(strpos($link, 'facebook.com/l.php?u=') !== false) + $link = urldecode(extractFromDelimiters($link, 'facebook.com/l.php?u=', '&')); + return ' href="' . $link . '"'; + } + }; + + //Utility function for converting facebook emoticons + $unescape_fb_emote = function($matches){ + static $facebook_emoticons = array( + 'smile' => ':)', + 'frown' => ':(', + 'tongue' => ':P', + 'grin' => ':D', + 'gasp' => ':O', + 'wink' => ';)', + 'pacman' => ':<', + 'grumpy' => '>_<', + 'unsure' => ':/', + 'cry' => ':\'(', + 'kiki' => '^_^', + 'glasses' => '8-)', + 'sunglasses' => 'B-)', + 'heart' => '<3', + 'devil' => ']:D', + 'angel' => '0:)', + 'squint' => '-_-', + 'confused' => 'o_O', + 'upset' => 'xD', + 'colonthree' => ':3', + 'like' => '👍'); + $len = count($matches); + if ($len > 1) + for ($i = 1; $i < $len; $i++) + foreach ($facebook_emoticons as $name => $emote) + if ($matches[$i] === $name) + return $emote; + return $matches[0]; + }; + + $html = null; + + //Handle captcha response sent by the viewer + if (isset($_POST['captcha_response'])) { + if (session_status() == PHP_SESSION_NONE) + session_start(); + if (isset($_SESSION['captcha_fields'], $_SESSION['captcha_action'])) { + $captcha_action = $_SESSION['captcha_action']; + $captcha_fields = $_SESSION['captcha_fields']; + $captcha_fields['captcha_response'] = preg_replace("/[^a-zA-Z0-9]+/", "", $_POST['captcha_response']); + $http_options = array( + 'http' => array( + 'method' => 'POST', + 'user_agent' => ini_get('user_agent'), + 'header' => array("Content-type: + application/x-www-form-urlencoded\r\nReferer: $captcha_action\r\nCookie: noscript=1\r\n"), + 'content' => http_build_query($captcha_fields) + ), + ); + $context = stream_context_create($http_options); + $html = getContents($captcha_action, false, $context); + + if($html === false) { + returnServerError('Failed to submit captcha response back to Facebook'); + } + unset($_SESSION['captcha_fields']); + $html = str_get_html($html); + } + unset($_SESSION['captcha_fields']); + unset($_SESSION['captcha_action']); + } + + //Retrieve page contents + if(is_null($html)) { + $http_options = array( + 'http' => array( + 'method' => 'GET', + 'user_agent' => ini_get('user_agent'), + 'header' => 'Accept-Language: ' . getEnv('HTTP_ACCEPT_LANGUAGE') . "\r\n" + ) + ); + $context = stream_context_create($http_options); + if(!strpos($this->getInput('u'), "/")) { + $html = getSimpleHTMLDOM(self::URI . urlencode($this->getInput('u')) . '?_fb_noscript=1', + false, + $context) + or returnServerError('No results for this query.'); + } else { + $html = getSimpleHTMLDOM(self::URI . 'pages/' . $this->getInput('u') . '?_fb_noscript=1', + false, + $context) + or returnServerError('No results for this query.'); + } + } + + //Handle captcha form? + $captcha = $html->find('div.captcha_interstitial', 0); + if (!is_null($captcha)) { + //Save form for submitting after getting captcha response + if (session_status() == PHP_SESSION_NONE) + session_start(); + $captcha_fields = array(); + foreach ($captcha->find('input, button') as $input) + $captcha_fields[$input->name] = $input->value; + $_SESSION['captcha_fields'] = $captcha_fields; + $_SESSION['captcha_action'] = $captcha->find('form', 0)->action; + + //Show captcha filling form to the viewer, proxying the captcha image + $img = base64_encode(getContents($captcha->find('img', 0)->src)); + header('HTTP/1.1 500 ' . Http::getMessageForCode(500)); + header('Content-Type: text/html'); + $message = << +

    Facebook captcha challenge

    +

    Unfortunately, rss-bridge cannot fetch the requested page.
    + Facebook wants rss-bridge to resolve the following captcha:

    +

    +

    Response: +

    + +EOD; + die($message); + } + + //No captcha? We can carry on retrieving page contents :) + $element = $html + ->find('#pagelet_timeline_main_column')[0] + ->children(0) + ->children(0) + ->children(0) + ->next_sibling() + ->children(0); + + if(isset($element)) { + + $author = str_replace(' | Facebook', '', $html->find('title#pageTitle', 0)->innertext); + $profilePic = 'https://graph.facebook.com/' + . $this->getInput('u') + . '/picture?width=200&height=200'; + + $this->authorName = $author; + + foreach($element->children() as $cell) { + // Manage summary posts + if(strpos($cell->class, '_3xaf') !== false) { + $posts = $cell->children(); + } else { + $posts = array($cell); + } + + foreach($posts as $post) { + // Check media type + switch($this->getInput('media_type')) { + case 'all': break; + case 'video': + if(empty($post->find('[aria-label=Video]'))) continue 2; + break; + case 'novideo': + if(!empty($post->find('[aria-label=Video]'))) continue 2; + break; + default: break; + } + + $item = array(); + + if(count($post->find('abbr')) > 0) { + + //Retrieve post contents + $content = preg_replace( + '/(?i)>
    ]+)>(.+?)div\ class=\"userContent\"/i', + '', + $post); + + $content = preg_replace( + '/(?i)>
    ]+)>(.+?)<\/div><\/div>
    ]+)>(.+?)div\ class=\"[^u]+userContent\"/i', + '', + $content); + + $content = preg_replace( + '/(?i)>
    ]+)>(.+?)<\/div>/i', + '', + $content); + + //Remove html nodes, keep only img, links, basic formatting + $content = strip_tags($content, '

    '); + + //Adapt link hrefs: convert relative links into absolute links and bypass external link redirection + $content = preg_replace_callback('/ href=\"([^"]+)\"/i', $unescape_fb_link, $content); + + //Clean useless html tag properties and fix link closing tags + foreach (array( + 'onmouseover', + 'onclick', + 'target', + 'ajaxify', + 'tabindex', + 'class', + 'style', + 'data-[^=]*', + 'aria-[^=]*', + 'role', + 'rel', + 'id') as $property_name) + $content = preg_replace('/ ' . $property_name . '=\"[^"]*\"/i', '', $content); + $content = preg_replace('/<\/a [^>]+>/i', '', $content); + + //Convert textual representation of emoticons eg + //"smile emoticon" back to ASCII emoticons eg ":)" + $content = preg_replace_callback( + '/([^ <>]+) ([^<>]+)<\/u><\/i>/i', + $unescape_fb_emote, + $content + ); + + //Retrieve date of the post + $date = $post->find("abbr")[0]; + if(isset($date) && $date->hasAttribute('data-utime')) { + $date = $date->getAttribute('data-utime'); + } else { + $date = 0; + } + + //Build title from username and content + $title = $author; + if(strlen($title) > 24) + $title = substr($title, 0, strpos(wordwrap($title, 24), "\n")) . '...'; + $title = $title . ' | ' . strip_tags($content); + if(strlen($title) > 64) + $title = substr($title, 0, strpos(wordwrap($title, 64), "\n")) . '...'; + + //Build and add final item + $item['uri'] = self::URI . $post->find('abbr')[0]->parent()->getAttribute('href'); + $item['content'] = $content; + $item['title'] = $title; + $item['author'] = $author; + $item['timestamp'] = $date; + $this->items[] = $item; + } + } + } + } + } + + public function getName(){ + if(!empty($this->authorName)) { + return isset($this->extraInfos['name']) ? $this->extraInfos['name'] : $this->authorName + . ' - Facebook Bridge'; + } + + return parent::getName(); + } +} diff --git a/bridges/FeedExpanderExampleBridge.php b/bridges/FeedExpanderExampleBridge.php new file mode 100644 index 0000000..9d2f178 --- /dev/null +++ b/bridges/FeedExpanderExampleBridge.php @@ -0,0 +1,62 @@ + array( + 'version' => array( + 'name' => 'Version', + 'type' => 'list', + 'required' => true, + 'title' => 'Select your feed format/version', + 'defaultValue' => 'RSS 2.0', + 'values' => array( + 'RSS 0.91' => 'rss_0_9_1', + 'RSS 1.0' => 'rss_1_0', + 'RSS 2.0' => 'rss_2_0', + 'ATOM 1.0' => 'atom_1_0' + ) + ) + ) + ); + + public function collectData(){ + switch($this->getInput('version')) { + case 'rss_0_9_1': + parent::collectExpandableDatas('http://static.userland.com/gems/backend/sampleRss.xml'); + break; + case 'rss_1_0': + parent::collectExpandableDatas('http://feeds.nature.com/nature/rss/current?format=xml'); + break; + case 'rss_2_0': + parent::collectExpandableDatas('http://feeds.rssboard.org/rssboard?format=xml'); + break; + case 'atom_1_0': + parent::collectExpandableDatas('http://segfault.linuxmint.com/feed/atom/'); + break; + default: returnClientError('Unknown version ' . $this->getInput('version') . '!'); + } + } + + protected function parseItem($newsItem) { + switch($this->getInput('version')) { + case 'rss_0_9_1': + return $this->parseRSS_0_9_1_Item($newsItem); + break; + case 'rss_1_0': + return $this->parseRSS_1_0_Item($newsItem); + break; + case 'rss_2_0': + return $this->parseRSS_2_0_Item($newsItem); + break; + case 'atom_1_0': + return $this->parseATOMItem($newsItem); + break; + default: returnClientError('Unknown version ' . $this->getInput('version') . '!'); + } + } +} diff --git a/bridges/FierPandaBridge.php b/bridges/FierPandaBridge.php new file mode 100644 index 0000000..cd9d11b --- /dev/null +++ b/bridges/FierPandaBridge.php @@ -0,0 +1,24 @@ +find('div.container-content article') as $element) { + $item = array(); + $item['uri'] = $this->getURI() . $element->find('a', 0)->href; + $item['title'] = trim($element->find('h1 a', 0)->innertext); + // Remove the link at the end of the article + $element->find('p a', 0)->outertext = ''; + $item['content'] = $element->find('p', 0)->innertext; + $this->items[] = $item; + } + } +} diff --git a/bridges/FilterBridge.php b/bridges/FilterBridge.php new file mode 100644 index 0000000..e8b451c --- /dev/null +++ b/bridges/FilterBridge.php @@ -0,0 +1,77 @@ + array( + 'name' => 'Feed URL', + 'required' => true, + ), + 'filter' => array( + 'name' => 'Filter item title (regular expression)', + 'required' => false, + ), + 'filter_type' => array( + 'name' => 'Filter type', + 'type' => 'list', + 'required' => false, + 'values' => array( + 'Permit' => 'permit', + 'Block' => 'block', + ), + 'defaultValue' => 'permit', + ), + )); + + protected function parseItem($newItem){ + $item = parent::parseItem($newItem); + + switch(true) { + case $this->getFilterType() === 'permit': + if (preg_match($this->getFilter(), $item['title'])) { + return $item; + } + break; + case $this->getFilterType() === 'block': + if (!preg_match($this->getFilter(), $item['title'])) { + return $item; + } + break; + } + return null; + } + + protected function getFilter(){ + return '/' . $this->getInput('filter') . '/'; + } + + protected function getFilterType(){ + return $this->getInput('filter_type'); + } + + public function getURI(){ + $url = $this->getInput('url'); + + if(empty($url)) { + $url = parent::getURI(); + } + return $url; + } + + public function collectData(){ + if($this->getInput('url') && substr($this->getInput('url'), 0, strlen('http')) !== 'http') { + // just in case someone find a way to access local files by playing with the url + returnClientError('The url parameter must either refer to http or https protocol.'); + } + try{ + $this->collectExpandableDatas($this->getURI()); + } catch (HttpException $e) { + $this->collectExpandableDatas($this->getURI()); + } + } +} diff --git a/bridges/FlickrBridge.php b/bridges/FlickrBridge.php new file mode 100644 index 0000000..f5ebe9c --- /dev/null +++ b/bridges/FlickrBridge.php @@ -0,0 +1,120 @@ + array(), + 'By keyword' => array( + 'q' => array( + 'name' => 'Keyword', + 'type' => 'text', + 'required' => true, + 'title' => 'Insert keyword', + 'exampleValue' => 'bird' + ) + ), + 'By username' => array( + 'u' => array( + 'name' => 'Username', + 'type' => 'text', + 'required' => true, + 'title' => 'Insert username (as shown in the address bar)', + 'exampleValue' => 'flickr' + ) + ), + ); + + public function collectData(){ + switch($this->queriedContext) { + case 'Explore': + $key = 'photos'; + $html = getSimpleHTMLDOM(self::URI . 'explore') + or returnServerError('Could not request Flickr.'); + break; + case 'By keyword': + $key = 'photos'; + $html = getSimpleHTMLDOM(self::URI . 'search/?q=' . urlencode($this->getInput('q')) . '&s=rec') + or returnServerError('No results for this query.'); + break; + case 'By username': + $key = 'photoPageList'; + $html = getSimpleHTMLDOM(self::URI . 'photos/' . urlencode($this->getInput('u'))) + or returnServerError('Requested username can\'t be found.'); + break; + default: + returnClientError('Invalid context: ' . $this->queriedContext); + } + + // Find SCRIPT containing JSON data + $model = $html->find('.modelExport', 0); + $model_text = $model->innertext; + + // Find start and end of JSON data + $start = strpos($model_text, 'modelExport:') + strlen('modelExport:'); + $end = strpos($model_text, 'auth:') - strlen('auth:'); + + // Dissect JSON data and remove trailing comma + $model_text = trim(substr($model_text, $start, $end - $start)); + $model_text = substr($model_text, 0, strlen($model_text) - 1); + + $model_json = json_decode($model_text, true); + + foreach($html->find('.photo-list-photo-view') as $element) { + // Get the styles + $style = explode(';', $element->style); + + // Get the background-image style + $backgroundImage = explode(':', end($style)); + + // URI type : url(//cX.staticflickr.com/X/XXXXX/XXXXXXXXX.jpg) + $imageURI = trim(str_replace(['url(', ')'], '', end($backgroundImage))); + + // Get the image ID + $imageURIs = explode('_', basename($imageURI)); + $imageID = reset($imageURIs); + + // Use JSON data to build items + foreach(reset($model_json)[0][$key]['_data'] as $element) { + if($element['id'] === $imageID) { + $item = array(); + + /* Author name depends on scope. On a keyword search the + * author is part of the picture data. On a username search + * the author is part of the owner data. + */ + if(array_key_exists('username', $element)) { + $item['author'] = $element['username']; + } elseif (array_key_exists('owner', reset($model_json)[0])) { + $item['author'] = reset($model_json)[0]['owner']['username']; + } + + $item['title'] = (array_key_exists('title', $element) ? $element['title'] : 'Untitled'); + $item['uri'] = self::URI . 'photo.gne?id=' . $imageID; + + $description = (array_key_exists('description', $element) ? $element['description'] : ''); + + $item['content'] = '

    ' + . $description + . '

    '; + + $this->items[] = $item; + + break; + } + } + } + } +} diff --git a/bridges/FootitoBridge.php b/bridges/FootitoBridge.php new file mode 100644 index 0000000..ac06cd5 --- /dev/null +++ b/bridges/FootitoBridge.php @@ -0,0 +1,75 @@ +find('div.post') as $element) { + $item = array(); + + $content = trim($element->innertext); + $content = str_replace( + "find('.contenu .texte ', 0)->plaintext; + $item['title'] = $title; + + $info = $element->find('div.infos', 0); + + $item['timestamp'] = strtotime($info->find('time', 0)->datetime); + $item['author'] = $info->find('a.auteur', 0)->plaintext; + + $this->items[] = $item; + } + } +} diff --git a/bridges/FourchanBridge.php b/bridges/FourchanBridge.php new file mode 100644 index 0000000..6aaa13e --- /dev/null +++ b/bridges/FourchanBridge.php @@ -0,0 +1,78 @@ + array( + 'name' => 'Thread category', + 'required' => true + ), + 't' => array( + 'name' => 'Thread number', + 'type' => 'number', + 'required' => true + ) + )); + + public function getURI(){ + if(!is_null($this->getInput('c')) && !is_null($this->getInput('t'))) { + return static::URI . $this->getInput('c') . '/thread/' . $this->getInput('t'); + } + + return parent::getURI(); + } + + public function collectData(){ + + $html = getSimpleHTMLDOM($this->getURI()) + or returnServerError("Could not request 4chan, thread not found"); + + foreach($html->find('div.postContainer') as $element) { + $item = array(); + $item['id'] = $element->find('.post', 0)->getAttribute('id'); + $item['uri'] = $this->getURI() . '#' . $item['id']; + $item['timestamp'] = $element->find('span.dateTime', 0)->getAttribute('data-utc'); + $item['author'] = $element->find('span.name', 0)->plaintext; + + $file = $element->find('.file', 0); + + if(!empty($file)) { + $item['image'] = $element->find('.file a', 0)->href; + $item['imageThumb'] = $element->find('.file img', 0)->src; + if(!isset($item['imageThumb']) and strpos($item['image'], '.swf') !== false) + $item['imageThumb'] = 'http://i.imgur.com/eO0cxf9.jpg'; + } + + if(!empty($element->find('span.subject', 0)->innertext)) { + $item['subject'] = $element->find('span.subject', 0)->innertext; + } + + $item['title'] = 'reply ' . $item['id'] . ' | ' . $item['author']; + if(isset($item['subject'])) { + $item['title'] = $item['subject'] . ' - ' . $item['title']; + } + + $content = $element->find('.postMessage', 0)->innertext; + $content = str_replace('href="#p', 'href="' . $this->getURI() . '#p', $content); + $item['content'] = '' . $content . ''; + + if(isset($item['image'])) { + $item['content'] = '' + . $item['id'] + . '
    ' + .$item['content']; + } + $this->items[] = $item; + } + $this->items = array_reverse($this->items); + } +} diff --git a/bridges/FuturaSciencesBridge.php b/bridges/FuturaSciencesBridge.php new file mode 100644 index 0000000..b9479c3 --- /dev/null +++ b/bridges/FuturaSciencesBridge.php @@ -0,0 +1,173 @@ + array( + 'name' => 'Feed', + 'type' => 'list', + 'values' => array( + 'Les flux multi-magazines' => array( + 'Les dernières actualités de Futura-Sciences' => 'actualites', + 'Les dernières définitions de Futura-Sciences' => 'definitions', + 'Les dernières photos de Futura-Sciences' => 'photos', + 'Les dernières questions - réponses de Futura-Sciences' => 'questions-reponses', + 'Les derniers dossiers de Futura-Sciences' => 'dossiers' + ), + 'Les flux Services' => array( + 'Les cartes virtuelles de Futura-Sciences' => 'services/cartes-virtuelles', + 'Les fonds d\'écran de Futura-Sciences' => 'services/fonds-ecran' + ), + 'Les flux Santé' => array( + 'Les dernières actualités de Futura-Santé' => 'sante/actualites', + 'Les dernières définitions de Futura-Santé' => 'sante/definitions', + 'Les dernières questions-réponses de Futura-Santé' => 'sante/question-reponses', + 'Les derniers dossiers de Futura-Santé' => 'sante/dossiers' + ), + 'Les flux High-Tech' => array( + 'Les dernières actualités de Futura-High-Tech' => 'high-tech/actualites', + 'Les dernières astuces de Futura-High-Tech' => 'high-tech/question-reponses', + 'Les dernières définitions de Futura-High-Tech' => 'high-tech/definitions', + 'Les derniers dossiers de Futura-High-Tech' => 'high-tech/dossiers' + ), + 'Les flux Espace' => array( + 'Les dernières actualités de Futura-Espace' => 'espace/actualites', + 'Les dernières définitions de Futura-Espace' => 'espace/definitions', + 'Les dernières questions-réponses de Futura-Espace' => 'espace/question-reponses', + 'Les derniers dossiers de Futura-Espace' => 'espace/dossiers' + ), + 'Les flux Environnement' => array( + 'Les dernières actualités de Futura-Environnement' => 'environnement/actualites', + 'Les dernières définitions de Futura-Environnement' => 'environnement/definitions', + 'Les dernières questions-réponses de Futura-Environnement' => 'environnement/question-reponses', + 'Les derniers dossiers de Futura-Environnement' => 'environnement/dossiers' + ), + 'Les flux Maison' => array( + 'Les dernières actualités de Futura-Maison' => 'maison/actualites', + 'Les dernières astuces de Futura-Maison' => 'maison/question-reponses', + 'Les dernières définitions de Futura-Maison' => 'maison/definitions', + 'Les derniers dossiers de Futura-Maison' => 'maison/dossiers' + ), + 'Les flux Nature' => array( + 'Les dernières actualités de Futura-Nature' => 'nature/actualites', + 'Les dernières définitions de Futura-Nature' => 'nature/definitions', + 'Les dernières questions-réponses de Futura-Nature' => 'nature/question-reponses', + 'Les derniers dossiers de Futura-Nature' => 'nature/dossiers' + ), + 'Les flux Terre' => array( + 'Les dernières actualités de Futura-Terre' => 'terre/actualites', + 'Les dernières définitions de Futura-Terre' => 'terre/definitions', + 'Les dernières questions-réponses de Futura-Terre' => 'terre/question-reponses', + 'Les derniers dossiers de Futura-Terre' => 'terre/dossiers' + ), + 'Les flux Matière' => array( + 'Les dernières actualités de Futura-Matière' => 'matiere/actualites', + 'Les dernières définitions de Futura-Matière' => 'matiere/definitions', + 'Les dernières questions-réponses de Futura-Matière' => 'matiere/question-reponses', + 'Les derniers dossiers de Futura-Matière' => 'matiere/dossiers' + ), + 'Les flux Mathématiques' => array( + 'Les dernières actualités de Futura-Mathématiques' => 'mathematiques/actualites', + 'Les derniers dossiers de Futura-Mathématiques' => 'mathematiques/dossiers' + ) + ) + ) + )); + + public function collectData(){ + $url = self::URI . 'rss/' . $this->getInput('feed') . '.xml'; + $this->collectExpandableDatas($url, 10); + } + + protected function parseItem($newsItem){ + $item = parent::parseItem($newsItem); + $item['uri'] = str_replace('#xtor=RSS-8', '', $item['uri']); + $article = getSimpleHTMLDOMCached($item['uri']) + or returnServerError('Could not request Futura-Sciences: ' . $item['uri']); + $item['content'] = $this->extractArticleContent($article); + $author = $this->extractAuthor($article); + $item['author'] = empty($author) ? $item['author'] : $author; + return $item; + } + + private function stripWithDelimiters($string, $start, $end){ + while(strpos($string, $start) !== false) { + $section_to_remove = substr($string, strpos($string, $start)); + $section_to_remove = substr($section_to_remove, 0, strpos($section_to_remove, $end) + strlen($end)); + $string = str_replace($section_to_remove, '', $string); + } return $string; + } + + private function stripRecursiveHTMLSection($string, $tag_name, $tag_start){ + $open_tag = '<' . $tag_name; + $close_tag = ''; + $close_tag_length = strlen($close_tag); + if(strpos($tag_start, $open_tag) === 0) { + while(strpos($string, $tag_start) !== false) { + $max_recursion = 100; + $section_to_remove = null; + $section_start = strpos($string, $tag_start); + $search_offset = $section_start; + do { + $max_recursion--; + $section_end = strpos($string, $close_tag, $search_offset); + $search_offset = $section_end + $close_tag_length; + $section_to_remove = substr($string, $section_start, $section_end - $section_start + $close_tag_length); + $open_tag_count = substr_count($section_to_remove, $open_tag); + $close_tag_count = substr_count($section_to_remove, $close_tag); + } while ($open_tag_count > $close_tag_count && $max_recursion > 0); + $string = str_replace($section_to_remove, '', $string); + } + } + return $string; + } + + private function extractArticleContent($article){ + $contents = $article->find('section.article-text-classic', 0)->innertext; + $headline = trim($article->find('p.description', 0)->plaintext); + if(!empty($headline)) + $headline = '

    ' . $headline . '

    '; + + foreach (array( + '
    stripRecursiveHTMLSection($contents, 'div', $div_start); + } + + $contents = $this->stripWithDelimiters($contents, '
    '); + $contents = $this->stripWithDelimiters($contents, '

    '); + $contents = $this->stripWithDelimiters($contents, 'fs:definition="', '"'); + $contents = $this->stripWithDelimiters($contents, 'fs:xt:clicktype="', '"'); + $contents = $this->stripWithDelimiters($contents, 'fs:xt:clickname="', '"'); + $contents = $this->stripWithDelimiters($contents, ''); + return $content; + } + + private function fetchPostContent($uri, $site_url){ + $html = getSimpleHTMLDOM($uri); + if(!$html) { + return 'Could not request GBAtemp ' . $uri; + } + + $content = $html->find('div.messageContent', 0)->innertext; + return $this->cleanupPostContent($content, $site_url); + } + + public function collectData(){ + + $html = getSimpleHTMLDOM(self::URI) + or returnServerError('Could not request GBAtemp.'); + + switch($this->getInput('type')) { + case 'N': + foreach($html->find('li[class=news_item full]') as $newsItem) { + $url = self::URI . $newsItem->find('a', 0)->href; + $time = intval( + $this->extractFromDelimiters( + $newsItem->find('abbr.DateTime', 0)->outertext, + 'data-time="', + '"' + ) + ); + $author = $newsItem->find('a.username', 0)->plaintext; + $title = $newsItem->find('a', 1)->plaintext; + $content = $this->fetchPostContent($url, self::URI); + $this->items[] = $this->buildItem($url, $title, $author, $time, $content); + } + case 'R': + foreach($html->find('li.portal_review') as $reviewItem) { + $url = self::URI . $reviewItem->find('a', 0)->href; + $title = $reviewItem->find('span.review_title', 0)->plaintext; + $content = getSimpleHTMLDOM($url) + or returnServerError('Could not request GBAtemp: ' . $uri); + $author = $content->find('a.username', 0)->plaintext; + $time = intval( + $this->extractFromDelimiters( + $content->find('abbr.DateTime', 0)->outertext, + 'data-time="', + '"' + ) + ); + $intro = '

    ' . ($content->find('div#review_intro', 0)->plaintext) . '

    '; + $review = $content->find('div#review_main', 0)->innertext; + $subheader = '

    ' . $content->find('div.review_subheader', 0)->plaintext . '

    '; + $procons = $content->find('table.review_procons', 0)->outertext; + $scores = $content->find('table.reviewscores', 0)->outertext; + $content = $this->cleanupPostContent($intro . $review . $subheader . $procons . $scores, self::URI); + $this->items[] = $this->buildItem($url, $title, $author, $time, $content); + } + case 'T': + foreach($html->find('li.portal-tutorial') as $tutorialItem) { + $url = self::URI . $tutorialItem->find('a', 0)->href; + $title = $tutorialItem->find('a', 0)->plaintext; + $time = intval( + $this->extractFromDelimiters( + $tutorialItem->find('abbr.DateTime', 0)->outertext, + 'data-time="', + '"' + ) + ); + $author = $tutorialItem->find('a.username', 0)->plaintext; + $content = $this->fetchPostContent($url, self::URI); + $this->items[] = $this->buildItem($url, $title, $author, $time, $content); + } + case 'F': + foreach($html->find('li.rc_item') as $postItem) { + $url = self::URI . $postItem->find('a', 1)->href; + $title = $postItem->find('a', 1)->plaintext; + $time = intval( + $this->extractFromDelimiters( + $postItem->find('abbr.DateTime', 0)->outertext, + 'data-time="', + '"' + ) + ); + $author = $postItem->find('a.username', 0)->plaintext; + $content = $this->fetchPostContent($url, self::URI); + $this->items[] = $this->buildItem($url, $title, $author, $time, $content); + } + } + } + + public function getName() { + if(!is_null($this->getInput('type'))) { + $type = array_search( + $this->getInput('type'), + self::PARAMETERS[$this->queriedContext]['type']['values'] + ); + return 'GBAtemp ' . $type . ' Bridge'; + } + + return parent::getName(); + } +} diff --git a/bridges/GelbooruBridge.php b/bridges/GelbooruBridge.php new file mode 100644 index 0000000..fa4ce11 --- /dev/null +++ b/bridges/GelbooruBridge.php @@ -0,0 +1,22 @@ +getURI() + . 'index.php?page=post&s=list&pid=' + . ($this->getInput('p') ? ($this->getInput('p') - 1) * static::PIDBYPAGE : '') + . '&tags=' . urlencode($this->getInput('t')); + } +} diff --git a/bridges/GiphyBridge.php b/bridges/GiphyBridge.php new file mode 100644 index 0000000..26d1eba --- /dev/null +++ b/bridges/GiphyBridge.php @@ -0,0 +1,76 @@ + array( + 'name' => 'search tag', + 'required' => true + ), + 'n' => array( + 'name' => 'max number of returned items', + 'type' => 'number' + ) + )); + + public function collectData(){ + $html = ''; + $base_url = 'http://giphy.com'; + $html = getSimpleHTMLDOM(self::URI . '/search/' . urlencode($this->getInput('s') . '/')) + or returnServerError('No results for this query.'); + + $max = GIPHY_LIMIT; + if($this->getInput('n')) { + $max = $this->getInput('n'); + } + + $limit = 0; + $kw = urlencode($this->getInput('s')); + foreach($html->find('div.hoverable-gif') as $entry) { + if($limit < $max) { + $node = $entry->first_child(); + $href = $node->getAttribute('href'); + + $html2 = getSimpleHTMLDOM(self::URI . $href) + or returnServerError('No results for this query.'); + $figure = $html2->getElementByTagName('figure'); + $img = $figure->firstChild(); + $caption = $figure->lastChild(); + + $item = array(); + $item['id'] = $img->getAttribute('data-gif_id'); + $item['uri'] = $img->getAttribute('data-bitly_gif_url'); + $item['username'] = 'Giphy - ' . ucfirst($kw); + $title = $caption->innertext(); + $title = preg_replace('/\s+/', ' ', $title); + $title = str_replace('animated GIF', '', $title); + $title = str_replace($kw, '', $title); + $title = preg_replace('/\s+/', ' ', $title); + $title = trim($title); + if(strlen($title) <= 0) { + $title = $item['id']; + } + $item['title'] = trim($title); + $item['content'] = ''; + + $this->items[] = $item; + $limit++; + } + } + } +} diff --git a/bridges/GithubIssueBridge.php b/bridges/GithubIssueBridge.php new file mode 100644 index 0000000..4f121d8 --- /dev/null +++ b/bridges/GithubIssueBridge.php @@ -0,0 +1,192 @@ + array( + 'u' => array( + 'name' => 'User name', + 'required' => true + ), + 'p' => array( + 'name' => 'Project name', + 'required' => true + ) + ), + 'Project Issues' => array( + 'c' => array( + 'name' => 'Show Issues Comments', + 'type' => 'checkbox' + ) + ), + 'Issue comments' => array( + 'i' => array( + 'name' => 'Issue number', + 'type' => 'number', + 'required' => 'true' + ) + ) + ); + + public function getName(){ + $name = $this->getInput('u') . '/' . $this->getInput('p'); + switch($this->queriedContext) { + case 'Project Issues': + if($this->getInput('c')) { + $prefix = static::NAME . 's comments for '; + } else { + $prefix = static::NAME . 's for '; + } + $name = $prefix . $name; + break; + case 'Issue comments': + $name = static::NAME . ' ' . $name . ' #' . $this->getInput('i'); + break; + default: return parent::getName(); + } + return $name; + } + + public function getURI(){ + if(!is_null($this->getInput('u')) && !is_null($this->getInput('p'))) { + $uri = static::URI . $this->getInput('u') . '/' . $this->getInput('p') . '/issues'; + if($this->queriedContext === 'Issue comments') { + $uri .= '/' . $this->getInput('i'); + } elseif($this->getInput('c')) { + $uri .= '?q=is%3Aissue+sort%3Aupdated-desc'; + } + return $uri; + } + + return parent::getURI(); + } + + protected function extractIssueComment($issueNbr, $title, $comment){ + $class = $comment->getAttribute('class'); + $classes = explode(' ', $class); + $event = false; + if(in_array('discussion-item', $classes)) { + $event = true; + } + + $author = 'unknown'; + if($comment->find('.author', 0)) { + $author = $comment->find('.author', 0)->plaintext; + } + + $uri = static::URI . $this->getInput('u') . '/' . $this->getInput('p') . '/issues/' . $issueNbr; + + $comment = $comment->firstChild(); + if(!$event) { + $comment = $comment->nextSibling(); + } + + if($event) { + $title .= ' / ' . substr($class, strpos($class, 'discussion-item-') + strlen('discussion-item-')); + if(!$comment->hasAttribute('id')) { + $items = array(); + $timestamp = strtotime($comment->find('relative-time', 0)->getAttribute('datetime')); + $content = $comment->innertext; + while($comment = $comment->nextSibling()) { + $item = array(); + $item['author'] = $author; + $item['title'] = html_entity_decode($title, ENT_QUOTES, 'UTF-8'); + $item['timestamp'] = $timestamp; + $item['content'] = $content . '

    ' . $comment->children(1)->innertext . '

    '; + $item['uri'] = $uri . '#' . $comment->children(1)->getAttribute('id'); + $items[] = $item; + } + return $items; + } + $content = $comment->parent()->innertext; + } else { + $title .= ' / ' . trim($comment->firstChild()->plaintext); + $content = "
    " . $comment->find('.comment-body', 0)->innertext . "
    "; + } + + $item = array(); + $item['author'] = $author; + $item['uri'] = $uri . '#' . $comment->getAttribute('id'); + $item['title'] = html_entity_decode($title, ENT_QUOTES, 'UTF-8'); + $item['timestamp'] = strtotime($comment->find('relative-time', 0)->getAttribute('datetime')); + $item['content'] = $content; + return $item; + } + + protected function extractIssueComments($issue){ + $items = array(); + $title = $issue->find('.gh-header-title', 0)->plaintext; + $issueNbr = trim(substr($issue->find('.gh-header-number', 0)->plaintext, 1)); + $comments = $issue->find('.js-discussion', 0); + foreach($comments->children() as $comment) { + $classes = explode(' ', $comment->getAttribute('class')); + if(in_array('discussion-item', $classes) + || in_array('timeline-comment-wrapper', $classes)) { + $item = $this->extractIssueComment($issueNbr, $title, $comment); + if(array_keys($item) !== range(0, count($item) - 1)) { + $item = array($item); + } + $items = array_merge($items, $item); + } + } + return $items; + } + + public function collectData(){ + $html = getSimpleHTMLDOM($this->getURI()) + or returnServerError('No results for Github Issue ' . $this->getURI()); + + switch($this->queriedContext) { + case 'Issue comments': + $this->items = $this->extractIssueComments($html); + break; + case 'Project Issues': + foreach($html->find('.js-active-navigation-container .js-navigation-item') as $issue) { + $info = $issue->find('.opened-by', 0); + $issueNbr = substr(trim($info->plaintext), 1, strpos(trim($info->plaintext), ' ')); + + $item = array(); + $item['content'] = ''; + + if($this->getInput('c')) { + $uri = static::URI . $this->getInput('u') . '/' . $this->getInput('p') . '/issues/' . $issueNbr; + $issue = getSimpleHTMLDOMCached($uri, static::CACHE_TIMEOUT); + if($issue) { + $this->items = array_merge($this->items, $this->extractIssueComments($issue)); + continue; + } + $item['content'] = 'Can not extract comments from ' . $uri; + } + + $item['author'] = $info->find('a', 0)->plaintext; + $item['timestamp'] = strtotime($info->find('relative-time', 0)->getAttribute('datetime')); + $item['title'] = html_entity_decode( + $issue->find('.js-navigation-open', 0)->plaintext, + ENT_QUOTES, + 'UTF-8' + ); + $comments = $issue->find('.col-5', 0)->plaintext; + $item['content'] .= "\n" . 'Comments: ' . ($comments ? $comments : '0'); + $item['uri'] = self::URI . $issue->find('.js-navigation-open', 0)->getAttribute('href'); + $this->items[] = $item; + } + break; + } + + array_walk($this->items, function(&$item){ + $item['content'] = preg_replace('/\s+/', ' ', $item['content']); + $item['content'] = str_replace('href="/', 'href="' . static::URI, $item['content']); + $item['content'] = str_replace( + 'href="#', + 'href="' . substr($item['uri'], 0, strpos($item['uri'], '#') + 1), + $item['content'] + ); + $item['title'] = preg_replace('/\s+/', ' ', $item['title']); + }); + } +} diff --git a/bridges/GithubSearchBridge.php b/bridges/GithubSearchBridge.php new file mode 100644 index 0000000..d3a615b --- /dev/null +++ b/bridges/GithubSearchBridge.php @@ -0,0 +1,50 @@ + array( + 'type' => 'text', + 'name' => 'Search query' + ) + )); + + public function collectData(){ + $params = array('utf8' => '✓', + 'q' => urlencode($this->getInput('s')), + 's' => 'updated', + 'o' => 'desc', + 'type' => 'Repositories'); + $url = self::URI . 'search?' . http_build_query($params); + + $html = getSimpleHTMLDOM($url) + or returnServerError('Error while downloading the website content'); + + foreach($html->find('div.repo-list-item') as $element) { + $item = array(); + + $uri = $element->find('h3 a', 0)->href; + $uri = substr(self::URI, 0, -1) . $uri; + $item['uri'] = $uri; + + $title = $element->find('h3', 0)->plaintext; + $item['title'] = $title; + + if (count($element->find('p')) == 2) { + $content = $element->find('p', 0)->innertext; + } else{ + $content = ''; + } + $item['content'] = $content; + + $date = $element->find('relative-time', 0)->datetime; + $item['timestamp'] = strtotime($date); + + $this->items[] = $item; + } + } +} diff --git a/bridges/GizmodoBridge.php b/bridges/GizmodoBridge.php new file mode 100644 index 0000000..35f162b --- /dev/null +++ b/bridges/GizmodoBridge.php @@ -0,0 +1,36 @@ +find('div.entry-content', 0)->innertext; + foreach($articleHTMLContent->find('pagespeed_iframe') as $element) { + $text .= '

    link to a iframe (could be a video): ' + . $element->src + . '


    '; + } + + $text = strip_tags($text, '

    '); + } + + $item['content'] = $text; + return $item; + } + + public function collectData(){ + $this->collectExpandableDatas('http://feeds.gawker.com/gizmodo/full'); + } +} diff --git a/bridges/GoComicsBridge.php b/bridges/GoComicsBridge.php new file mode 100644 index 0000000..27621d8 --- /dev/null +++ b/bridges/GoComicsBridge.php @@ -0,0 +1,59 @@ + array( + 'name' => 'comicname', + 'type' => 'text', + 'required' => true + ) + )); + + public function collectData(){ + $html = getSimpleHTMLDOM($this->getURI()) + or returnServerError('Could not request GoComics: ' . $this->getURI()); + + foreach($html->find('div.item-comic-container') as $element) { + + $img = $element->find('img', 0); + $link = $element->find('a.item-comic-link', 0); + $comic = $img->src; + $title = $link->title; + $url = $html->find('input.js-copy-link', 0)->value; + $date = substr($title, -10); + if (empty($title)) + $title = 'GoComics ' . $this->getInput('comicname') . ' on ' . $date; + $date = strtotime($date); + + $item = array(); + $item['id'] = $url; + $item['uri'] = $url; + $item['title'] = $title; + $item['author'] = preg_replace('/by /', '', $element->find('a.link-blended small', 0)->plaintext); + $item['timestamp'] = $date; + $item['content'] = '' . $title . ''; + $this->items[] = $item; + } + } + + public function getURI(){ + if(!is_null($this->getInput('comicname'))) { + return self::URI . urlencode($this->getInput('comicname')); + } + + return parent::getURI(); + } + + public function getName(){ + if(!is_null($this->getInput('comicname'))) { + return $this->getInput('comicname') . ' - GoComics'; + } + + return parent::getName(); + } +} diff --git a/bridges/GooglePlusPostBridge.php b/bridges/GooglePlusPostBridge.php new file mode 100644 index 0000000..0fccc96 --- /dev/null +++ b/bridges/GooglePlusPostBridge.php @@ -0,0 +1,111 @@ + array( + 'name' => 'username or Id', + 'required' => true + ) + )); + + public function collectData(){ + $username = $this->getInput('username'); + + // Usernames start with a + if it's not an ID + if(!is_numeric($username) && substr($username, 0, 1) !== '+') { + $username = '+' . $username; + } + + // get content parsed + $html = getSimpleHTMLDOMCached(self::URI . urlencode($username) . '/posts') + or returnServerError('No results for this query.'); + + // get title, url, ... there is a lot of intresting stuff in meta + $this->_title = $html->find('meta[property=og:title]', 0)->getAttribute('content'); + $this->_url = $html->find('meta[property=og:url]', 0)->getAttribute('content'); + + // I don't even know where to start with this discusting html... + foreach($html->find('div[jsname=WsjYwc]') as $post) { + $item = array(); + + $item['author'] = $item['fullname'] = $post->find('div div div div a', 0)->innertext; + $item['id'] = $post->find('div div div', 0)->getAttribute('id'); + $item['avatar'] = $post->find('div img', 0)->src; + $item['uri'] = self::URI . $post->find('div div div a', 1)->href; + + $timestamp = $post->find('a.qXj2He span', 0); + + if($timestamp) { + $item['timestamp'] = strtotime('+' . preg_replace( + '/[^0-9A-Za-z]/', + '', + $timestamp->getAttribute('aria-label'))); + } + + // hashtag to treat : https://plus.google.com/explore/tag + // $hashtags = array(); + // foreach($post->find('a.d-s') as $hashtag){ + // $hashtags[trim($hashtag->plaintext)] = self::URI . $hashtag->href; + // } + + $item['content'] = ''; + + // avatar display + $item['content'] .= '
    ' + . $item['author'] + . '
    '; + + $content = $post->find('div[jsname=EjRJtf]', 0); + // extract plaintext + $item['content_simple'] = $content->plaintext; + $item['title'] = substr($item['content_simple'], 0, 72) . '...'; + + // XXX ugly but I don't have any idea how to do a better stuff, + // str_replace on link doesn't work as expected and ask too many checks + foreach($content->find('a') as $link) { + $hasHttp = strpos($link->href, 'http'); + $hasDoubleSlash = strpos($link->href, '//'); + + if((!$hasHttp && !$hasDoubleSlash) + || (false !== $hasHttp && strpos($link->href, 'http') != 0) + || (false === $hasHttp && false !== $hasDoubleSlash && $hasDoubleSlash != 0)) { + // skipp bad link, for some hashtag or other stuff + if(strpos($link->href, '/') == 0) { + $link->href = substr($link->href, 1); + } + + $link->href = self::URI . $link->href; + } + } + $content = $content->innertext; + + $item['content'] .= '
    ' . $content . '
    '; + $item['content'] = trim(strip_tags($item['content'], '

    ')); + + $this->items[] = $item; + } + } + + public function getName(){ + return $this->_title ?: 'Google Plus Post Bridge'; + } + + public function getURI(){ + return $this->_url ?: parent::getURI(); + } +} diff --git a/bridges/GoogleSearchBridge.php b/bridges/GoogleSearchBridge.php new file mode 100644 index 0000000..2c4a5f1 --- /dev/null +++ b/bridges/GoogleSearchBridge.php @@ -0,0 +1,64 @@ + array( + 'name' => "keyword", + 'required' => true + ) + )); + + public function collectData(){ + $html = ''; + + $html = getSimpleHTMLDOM(self::URI + . 'search?q=' + . urlencode($this->getInput('q')) + .'&num=100&complete=0&tbs=qdr:y,sbd:1') + or returnServerError('No results for this query.'); + + $emIsRes = $html->find('div[id=ires]', 0); + + if(!is_null($emIsRes)) { + foreach($emIsRes->find('div[class=g]') as $element) { + + $item = array(); + + // Extract direct URL from google href (eg. /url?q=...) + $t = $element->find('a[href]', 0)->href; + $item['uri'] = '' . $t; + parse_str(parse_url($t, PHP_URL_QUERY), $parameters); + if(isset($parameters['q'])) { + $item['uri'] = $parameters['q']; + } + + $item['title'] = $element->find('h3', 0)->plaintext; + $item['content'] = $element->find('span[class=st]', 0)->plaintext; + + $this->items[] = $item; + } + } + } + + public function getName(){ + if(!is_null($this->getInput('q'))) { + return $this->getInput('q') . ' - Google search'; + } + + return parent::getName(); + } +} diff --git a/bridges/HDWallpapersBridge.php b/bridges/HDWallpapersBridge.php new file mode 100644 index 0000000..cea6e34 --- /dev/null +++ b/bridges/HDWallpapersBridge.php @@ -0,0 +1,83 @@ + array( + 'name' => 'category', + 'defaultValue' => 'latest_wallpapers' + ), + 'm' => array( + 'name' => 'max number of wallpapers' + ), + 'r' => array( + 'name' => 'resolution', + 'defaultValue' => '1920x1200', + 'exampleValue' => '1920x1200, 1680x1050,…' + ) + )); + + public function collectData(){ + $category = $this->category; + if(strrpos($category, 'wallpapers') !== strlen($category) - strlen('wallpapers')) { + $category .= '-desktop-wallpapers'; + } + + $num = 0; + $max = $this->getInput('m') ?: 14; + $lastpage = 1; + + for($page = 1; $page <= $lastpage; $page++) { + $link = self::URI . '/' . $category . '/page/' . $page; + $html = getSimpleHTMLDOM($link) + or returnServerError('No results for this query.'); + + if($page === 1) { + preg_match('/page\/(\d+)$/', $html->find('.pagination a', -2)->href, $matches); + $lastpage = min($matches[1], ceil($max / 14)); + } + + foreach($html->find('.wallpapers .wall a') as $element) { + $thumbnail = $element->find('img', 0); + + $item = array(); + // http://www.hdwallpapers.in/download/yosemite_reflections-1680x1050.jpg + $item['uri'] = self::URI + . '/download' + . str_replace('wallpapers.html', $this->getInput('r') . '.jpg', $element->href); + + $item['timestamp'] = time(); + $item['title'] = $element->find('p', 0)->text(); + $item['content'] = $item['title'] + . '
    '; + + $this->items[] = $item; + + $num++; + if ($num >= $max) + break 2; + } + } + } + + public function getName(){ + if(!is_null($this->getInput('c')) && !is_null($this->getInput('r'))) { + return 'HDWallpapers - ' + . str_replace(['__', '_'], [' & ', ' '], $this->getInput('c')) + . ' [' + . $this->getInput('r') + . ']'; + } + + return parent::getName(); + } +} diff --git a/bridges/HentaiHavenBridge.php b/bridges/HentaiHavenBridge.php new file mode 100644 index 0000000..21a0ff5 --- /dev/null +++ b/bridges/HentaiHavenBridge.php @@ -0,0 +1,37 @@ +find('div.zoe-grid') as $element) { + $item = array(); + $item['uri'] = $element->find('div.brick-content h3 a', 0)->href; + $thumbnailUri = $element->find('a.thumbnail-image img', 0)->getAttribute('data-src'); + $item['title'] = mb_convert_encoding( + trim($element->find('div.brick-content h3 a', 0)->innertext), + 'UTF-8', + 'HTML-ENTITIES' + ); + + $item['tags'] = $element->find('div.oFlyout_bg div.oFlyout div.flyoutContent span.tags', 0)->plaintext; + $item['content'] = 'Tags: ' + . $item['tags'] + . '


    ' + . $element->find('div.oFlyout_bg div.oFlyout div.flyoutContent p.description', 0)->innertext; + + $this->items[] = $item; + } + } +} diff --git a/bridges/IdenticaBridge.php b/bridges/IdenticaBridge.php new file mode 100644 index 0000000..ef52998 --- /dev/null +++ b/bridges/IdenticaBridge.php @@ -0,0 +1,52 @@ + array( + 'name' => 'username', + 'required' => true + ) + )); + + public function collectData(){ + $html = getSimpleHTMLDOM($this->getURI()) + or returnServerError('Requested username can\'t be found.'); + + foreach($html->find('li.major') as $dent) { + $item = array(); + + // get dent link + $item['uri'] = html_entity_decode($dent->find('a', 0)->href); + + // extract dent timestamp + $item['timestamp'] = strtotime($dent->find('abbr.easydate', 0)->plaintext); + + // extract dent text + $item['content'] = trim($dent->find('div.activity-content', 0)->innertext); + $item['title'] = $this->getInput('u') . ' | ' . $item['content']; + $this->items[] = $item; + } + } + + public function getName(){ + if(!is_null($this->getInput('u'))) { + return $this->getInput('u') . ' - Identica Bridge'; + } + + return parent::getName(); + } + + public function getURI(){ + if(!is_null($this->getInput('u'))) { + return self::URI . urlencode($this->getInput('u')); + } + + return parent::getURI(); + } +} diff --git a/bridges/InstagramBridge.php b/bridges/InstagramBridge.php new file mode 100644 index 0000000..7ae6a45 --- /dev/null +++ b/bridges/InstagramBridge.php @@ -0,0 +1,93 @@ + array( + 'name' => 'username', + 'required' => true + ), + 'media_type' => array( + 'name' => 'Media type', + 'type' => 'list', + 'required' => false, + 'values' => array( + 'Both' => 'all', + 'Video' => 'video', + 'Picture' => 'picture' + ), + 'defaultValue' => 'all' + ) + )); + + public function collectData(){ + $html = getSimpleHTMLDOM($this->getURI()) + or returnServerError('Could not request Instagram.'); + + $innertext = null; + + foreach($html->find('script') as $script) { + if('' === $script->innertext) { + continue; + } + + $pos = strpos(trim($script->innertext), 'window._sharedData'); + if(0 !== $pos) { + continue; + } + + $innertext = $script->innertext; + break; + } + + $json = trim(substr($innertext, $pos + 18), ' =;'); + $data = json_decode($json); + + $userMedia = $data->entry_data->ProfilePage[0]->user->media->nodes; + + foreach($userMedia as $media) { + // Check media type + switch($this->getInput('media_type')) { + case 'all': break; + case 'video': + if($media->is_video === false) continue 2; + break; + case 'picture': + if($media->is_video === true) continue 2; + break; + default: break; + } + + $item = array(); + $item['uri'] = self::URI . 'p/' . $media->code . '/'; + $item['content'] = ''; + if (isset($media->caption)) { + $item['title'] = $media->caption; + } else { + $item['title'] = basename($media->display_src); + } + $item['timestamp'] = $media->date; + $this->items[] = $item; + } + } + + public function getName(){ + if(!is_null($this->getInput('u'))) { + return $this->getInput('u') . ' - Instagram Bridge'; + } + + return parent::getName(); + } + + public function getURI(){ + if(!is_null($this->getInput('u'))) { + return self::URI . urlencode($this->getInput('u')); + } + + return parent::getURI(); + } +} diff --git a/bridges/IsoHuntBridge.php b/bridges/IsoHuntBridge.php new file mode 100644 index 0000000..57038fc --- /dev/null +++ b/bridges/IsoHuntBridge.php @@ -0,0 +1,465 @@ + array( + 'latest_category' => array( + 'name' => 'Latest category', + 'type' => 'list', + 'required' => true, + 'title' => 'Select your category', + 'defaultValue' => 'news', + 'values' => array( + 'Hot Torrents' => 'hot_torrents', + 'News' => 'news', + 'Releases' => 'releases', + 'Torrents' => 'torrents' + ) + ) + ), + + /* + * Get feeds for one of the "torrent" categories + * Make sure to add new categories also to get_torrent_category_index($)! + * Elements are sorted by name ascending! + */ + 'By "Torrent" category' => array( + 'torrent_category' => array( + 'name' => 'Torrent category', + 'type' => 'list', + 'required' => true, + 'title' => 'Select your category', + 'defaultValue' => 'anime', + 'values' => array( + 'Adult' => 'adult', + 'Anime' => 'anime', + 'Books' => 'books', + 'Games' => 'games', + 'Movies' => 'movies', + 'Music' => 'music', + 'Other' => 'other', + 'Series & TV' => 'series_tv', + 'Software' => 'software' + ) + ), + 'torrent_popularity' => array( + 'name' => 'Sort by popularity', + 'type' => 'checkbox', + 'title' => 'Activate to receive results by popularity' + ) + ), + + /* + * Get feeds for a specific search request + */ + 'Search torrent by name' => array( + 'search_name' => array( + 'name' => 'Name', + 'required' => true, + 'title' => 'Insert your search query', + 'exampleValue' => 'Bridge' + ), + 'search_category' => array( + 'name' => 'Category', + 'type' => 'list', + 'title' => 'Select your category', + 'defaultValue' => 'all', + 'values' => array( + 'Adult' => 'adult', + 'All' => 'all', + 'Anime' => 'anime', + 'Books' => 'books', + 'Games' => 'games', + 'Movies' => 'movies', + 'Music' => 'music', + 'Other' => 'other', + 'Series & TV' => 'series_tv', + 'Software' => 'software' + ) + ) + ) + ); + + public function getURI(){ + $uri = self::URI; + switch($this->queriedContext) { + case 'By "Latest" category': + switch($this->getInput('latest_category')) { + case 'hot_torrents': + $uri .= 'statistic/hot/torrents'; + break; + case 'news': + break; + case 'releases': + $uri .= 'releases.php'; + break; + case 'torrents': + $uri .= 'latest.php'; + break; + } + break; + case 'By "Torrent" category': + $uri .= $this->buildCategoryUri( + $this->getInput('torrent_category'), + $this->getInput('torrent_popularity') + ); + break; + case 'Search torrent by name': + $category = $this->getInput('search_category'); + $uri .= $this->buildCategoryUri($category); + if($category !== 'movies') + $uri .= '&ihq=' . urlencode($this->getInput('search_name')); + break; + + default: parent::getURI(); + } + + return $uri; + } + + public function getName(){ + switch($this->queriedContext) { + case 'By "Latest" category': + $categoryName = array_search( + $this->getInput('latest_category'), + self::PARAMETERS['By "Latest" category']['latest_category']['values'] + ); + $name = 'Latest ' . $categoryName . ' - ' . self::NAME; + break; + case 'By "Torrent" category': + $categoryName = array_search( + $this->getInput('torrent_category'), + self::PARAMETERS['By "Torrent" category']['torrent_category']['values'] + ); + $name = 'Category: ' . $categoryName . ' - ' . self::NAME; + break; + case 'Search torrent by name': + $categoryName = array_search( + $this->getInput('search_category'), + self::PARAMETERS['Search torrent by name']['search_category']['values'] + ); + $name = 'Search: "' + . $this->getInput('search_name') + . '" in category: ' + . $categoryName . ' - ' + . self::NAME; + break; + default: return parent::getName(); + } + + return $name; + } + + public function collectData(){ + $html = $this->loadHtml($this->getURI()); + + switch($this->queriedContext) { + case 'By "Latest" category': + switch($this->getInput('latest_category')) { + case 'hot_torrents': + $this->getLatestHotTorrents($html); + break; + case 'news': + $this->getLatestNews($html); + break; + case 'releases': + case 'torrents': + $this->getLatestTorrents($html); + break; + } + break; + case 'By "Torrent" category': + if($this->getInput('torrent_category') === 'movies') { + // This one is special (content wise) + $this->getMovieTorrents($html); + } else { + $this->getLatestTorrents($html); + } + break; + case 'Search torrent by name': + if($this->getInput('search_category') === 'movies') { + // This one is special (content wise) + $this->getMovieTorrents($html); + } else { + $this->getLatestTorrents($html); + } + break; + } + } + + #region Helper functions for "Movie Torrents" + + private function getMovieTorrents($html){ + $container = $html->find('div#w0', 0); + if(!$container) + returnServerError('Unable to find torrent container!'); + + $torrents = $container->find('article'); + if(!$torrents) + returnServerError('Unable to find torrents!'); + + foreach($torrents as $torrent) { + + $anchor = $torrent->find('a', 0); + if(!$anchor) + returnServerError('Unable to find anchor!'); + + $date = $torrent->find('small', 0); + if(!$date) + returnServerError('Unable to find date!'); + + $item = array(); + + $item['uri'] = $this->fixRelativeUri($anchor->href); + $item['title'] = $anchor->title; + // $item['author'] = + $item['timestamp'] = strtotime($date->plaintext); + $item['content'] = $this->fixRelativeUri($torrent->innertext); + + $this->items[] = $item; + } + } + + #endregion + + #region Helper functions for "Latest Hot Torrents" + + private function getLatestHotTorrents($html){ + $container = $html->find('div#serps', 0); + if(!$container) + returnServerError('Unable to find torrent container!'); + + $torrents = $container->find('tr'); + if(!$torrents) + returnServerError('Unable to find torrents!'); + + // Remove first element (header row) + $torrents = array_slice($torrents, 1); + + foreach($torrents as $torrent) { + + $cell = $torrent->find('td', 0); + if(!$cell) + returnServerError('Unable to find cell!'); + + $element = $cell->find('a', 0); + if(!$element) + returnServerError('Unable to find element!'); + + $item = array(); + + $item['uri'] = $element->href; + $item['title'] = $element->plaintext; + // $item['author'] = + // $item['timestamp'] = + // $item['content'] = + + $this->items[] = $item; + } + } + + #endregion + + #region Helper functions for "Latest News" + + private function getLatestNews($html){ + $container = $html->find('div#postcontainer', 0); + if(!$container) + returnServerError('Unable to find post container!'); + + $posts = $container->find('div.index-post'); + if(!$posts) + returnServerError('Unable to find posts!'); + + foreach($posts as $post) { + $item = array(); + + $item['uri'] = $this->latestNewsExtractUri($post); + $item['title'] = $this->latestNewsExtractTitle($post); + $item['author'] = $this->latestNewsExtractAuthor($post); + $item['timestamp'] = $this->latestNewsExtractTimestamp($post); + $item['content'] = $this->latestNewsExtractContent($post); + + $this->items[] = $item; + } + } + + private function latestNewsExtractAuthor($post){ + $author = $post->find('small', 0); + if(!$author) + returnServerError('Unable to find author!'); + + // The author is hidden within a string like: 'Posted by {author} on {date}' + preg_match('/Posted\sby\s(.*)\son/i', $author->innertext, $matches); + + return $matches[1]; + } + + private function latestNewsExtractTimestamp($post){ + $date = $post->find('small', 0); + if(!$date) + returnServerError('Unable to find date!'); + + // The date is hidden within a string like: 'Posted by {author} on {date}' + preg_match('/Posted\sby\s.*\son\s(.*)/i', $date->innertext, $matches); + + $timestamp = strtotime($matches[1]); + + // Make sure date is not in the future (dates are given like 'Nov. 20' without year) + if($timestamp > time()) { + $timestamp = strtotime('-1 year', $timestamp); + } + + return $timestamp; + } + + private function latestNewsExtractTitle($post){ + $title = $post->find('a', 0); + if(!$title) + returnServerError('Unable to find title!'); + + return $title->plaintext; + } + + private function latestNewsExtractUri($post){ + $uri = $post->find('a', 0); + if(!$uri) + returnServerError('Unable to find uri!'); + + return $uri->href; + } + + private function latestNewsExtractContent($post){ + $content = $post->find('div', 0); + if(!$content) + returnServerError('Unable to find content!'); + + // Remove

    ...

    (title) + foreach($content->find('h2') as $element) { + $element->outertext = ''; + } + + // Remove ... (author) + foreach($content->find('small') as $element) { + $element->outertext = ''; + } + + return $content->innertext; + } + + #endregion + + #region Helper functions for "Latest Torrents", "Latest Releases" and "Torrent Category" + + private function getLatestTorrents($html){ + $container = $html->find('div#serps', 0); + if(!$container) + returnServerError('Unable to find torrent container!'); + + $torrents = $container->find('tr[data-key]'); + if(!$torrents) + returnServerError('Unable to find torrents!'); + + foreach($torrents as $torrent) { + $item = array(); + + $item['uri'] = $this->latestTorrentsExtractUri($torrent); + $item['title'] = $this->latestTorrentsExtractTitle($torrent); + $item['author'] = $this->latestTorrentsExtractAuthor($torrent); + $item['timestamp'] = $this->latestTorrentsExtractTimestamp($torrent); + $item['content'] = ''; // There is no valuable content + + $this->items[] = $item; + } + } + + private function latestTorrentsExtractTitle($torrent){ + $cell = $torrent->find('td.title-row', 0); + if(!$cell) + returnServerError('Unable to find title cell!'); + + $title = $cell->find('span', 0); + if(!$title) + returnServerError('Unable to find title!'); + + return $title->plaintext; + } + + private function latestTorrentsExtractUri($torrent){ + $cell = $torrent->find('td.title-row', 0); + if(!$cell) + returnServerError('Unable to find title cell!'); + + $uri = $cell->find('a', 0); + if(!$uri) + returnServerError('Unable to find uri!'); + + return $this->fixRelativeUri($uri->href); + } + + private function latestTorrentsExtractAuthor($torrent){ + $cell = $torrent->find('td.user-row', 0); + if(!$cell) + return; // No author + + $user = $cell->find('a', 0); + if(!$user) + returnServerError('Unable to find user!'); + + return $user->plaintext; + } + + private function latestTorrentsExtractTimestamp($torrent){ + $cell = $torrent->find('td.date-row', 0); + if(!$cell) + returnServerError('Unable to find date cell!'); + + return strtotime('-' . $cell->plaintext, time()); + } + + #endregion + + #region Generic helper functions + + private function loadHtml($uri){ + $html = getSimpleHTMLDOM($uri); + if(!$html) + returnServerError('Unable to load ' . $uri . '!'); + + return $html; + } + + private function fixRelativeUri($uri){ + return preg_replace('/\//i', self::URI, $uri, 1); + } + + private function buildCategoryUri($category, $order_popularity = false){ + switch($category) { + case 'anime': $index = 1; break; + case 'software' : $index = 2; break; + case 'games' : $index = 3; break; + case 'adult' : $index = 4; break; + case 'movies' : $index = 5; break; + case 'music' : $index = 6; break; + case 'other' : $index = 7; break; + case 'series_tv' : $index = 8; break; + case 'books': $index = 9; break; + case 'all': + default: $index = 0; break; + } + + return 'torrents/?iht=' . $index . '&ihs=' . ($order_popularity ? 1 : 0) . '&age=0'; + } + + #endregion +} diff --git a/bridges/JapanExpoBridge.php b/bridges/JapanExpoBridge.php new file mode 100644 index 0000000..c80bb24 --- /dev/null +++ b/bridges/JapanExpoBridge.php @@ -0,0 +1,100 @@ + array( + 'name' => 'Show full contents', + 'type' => 'checkbox', + ) + )); + + public function collectData(){ + + function frenchPubDateToTimestamp($date_to_parse) { + return strtotime( + strtr( + strtolower(str_replace('Publié le ', '', $date_to_parse)), + array( + 'janvier' => 'jan', + 'février' => 'feb', + 'mars' => 'march', + 'avril' => 'apr', + 'mai' => 'may', + 'juin' => 'jun', + 'juillet' => 'jul', + 'août' => 'aug', + 'septembre' => 'sep', + 'octobre' => 'oct', + 'novembre' => 'nov', + 'décembre' => 'dec' + ) + ) + ); + } + + $convert_article_images = function($matches){ + if(is_array($matches) && count($matches) > 1) { + return ''; + } + }; + + $html = getSimpleHTMLDOM(self::URI) + or returnServerError('Could not request JapanExpo: ' . self::URI); + $fullcontent = $this->getInput('mode'); + $count = 0; + + foreach($html->find('a._tile2') as $element) { + + $url = $element->href; + $thumbnail = 'http://s.japan-expo.com/katana/images/JES049/paris.png'; + preg_match('/url\(([^)]+)\)/', $element->find('img.rspvimgset', 0)->style, $img_search_result); + + if(count($img_search_result) >= 2) + $thumbnail = trim($img_search_result[1], "'"); + + if($fullcontent) { + if($count >= 5) { + break; + } + + $article_html = getSimpleHTMLDOMCached('Could not request JapanExpo: ' . $url); + $header = $article_html->find('header.pageHeadBox', 0); + $timestamp = strtotime($header->find('time', 0)->datetime); + $title_html = $header->find('div.section', 0)->next_sibling(); + $title = $title_html->plaintext; + $headings = $title_html->next_sibling()->outertext; + $article = $article_html->find('div.content', 0)->innertext; + $article = preg_replace_callback( + '/]+ style="[^\(]+\(\'([^\']+)\'[^>]+>/i', + $convert_article_images, + $article); + + $content = $headings . $article; + } else { + $date_text = $element->find('span.date', 0)->plaintext; + $timestamp = frenchPubDateToTimestamp($date_text); + $title = trim($element->find('span._title', 0)->plaintext); + $content = '
    ' + . $date_text + . '
    Lire l\'article'; + } + + $item = array(); + $item['uri'] = $url; + $item['title'] = $title; + $item['timestamp'] = $timestamp; + $item['content'] = $content; + $this->items[] = $item; + $count++; + } + } +} diff --git a/bridges/KATBridge.php b/bridges/KATBridge.php new file mode 100644 index 0000000..c4325a6 --- /dev/null +++ b/bridges/KATBridge.php @@ -0,0 +1,123 @@ + array( + 'name' => 'keywords, separated by semicolons', + 'exampleValue' => 'first list;second list;…', + 'required' => true + ), + 'crit' => array( + 'type' => 'list', + 'name' => 'Search type', + 'values' => array( + 'search' => 'search', + 'category' => 'cat', + 'user' => 'usr' + ) + ), + 'cat_check' => array( + 'type' => 'checkbox', + 'name' => 'Specify category for normal search ?', + ), + 'cat' => array( + 'name' => 'Category number', + 'exampleValue' => '100, 200… See KAT for category number' + ), + 'trusted' => array( + 'type' => 'checkbox', + 'name' => 'Only get results from Elite or Verified uploaders ?', + ), + )); + public function collectData(){ + function parseDateTimestamp($element){ + $guessedDate = strptime($element, '%d-%m-%Y %H:%M:%S'); + $timestamp = mktime( + $guessedDate['tm_hour'], + $guessedDate['tm_min'], + $guessedDate['tm_sec'], + $guessedDate['tm_mon'] + 1, + $guessedDate['tm_mday'], + $guessedDate['tm_year'] + 1900); + return $timestamp; + } + $catBool = $this->getInput('cat_check'); + if($catBool) { + $catNum = $this->getInput('cat'); + } + $critList = $this->getInput('crit'); + $trustedBool = $this->getInput('trusted'); + $keywordsList = explode(';', $this->getInput('q')); + foreach($keywordsList as $keywords) { + switch($critList) { + case 'search': + if($catBool == false) { + $html = getSimpleHTMLDOM( + self::URI . + 'torrents-search.php?search=' . + rawurlencode($keywords) + ) or returnServerError('Could not request KAT.'); + } else { + $html = getSimpleHTMLDOM( + self::URI . + 'torrents-search.php?search=' . + rawurlencode($keywords) . + '&cat=' . + rawurlencode($catNum) + ) or returnServerError('Could not request KAT.'); + } + break; + case 'cat': + $html = getSimpleHTMLDOM( + self::URI . + 'torrents.php?cat=' . + rawurlencode($keywords) + ) or returnServerError('Could not request KAT.'); + break; + case 'usr': + $html = getSimpleHTMLDOM( + self::URI . + 'account-details.php?id=' . + rawurlencode($keywords) + ) or returnServerError('Could not request KAT.'); + break; + } + if ($html->find('table.ttable_headinner', 0) == false) + returnServerError('No result for query ' . $keywords); + foreach($html->find('tr.t-row') as $element) { + if(!$trustedBool + || !is_null($element->find('i[title="Elite Uploader"]', 0)) + || !is_null($element->find('i[title="Verified Uploader"]', 0))) { + $item = array(); + $item['uri'] = self::URI . $element->find('a', 2)->href; + $item['id'] = self::URI . $element->find('a.cellMainLink', 0)->href; + $item['timestamp'] = parseDateTimestamp($element->find('td', 2)->plaintext); + $item['author'] = $element->find('a.plain', 0)->plaintext; + $item['title'] = $element->find('a.cellMainLink', 0)->plaintext; + $item['seeders'] = (int)$element->find('td', 3)->plaintext; + $item['leechers'] = (int)$element->find('td', 4)->plaintext; + $item['size'] = $element->find('td', 1)->plaintext; + $item['content'] = $item['title'] + . '
    size: ' + . $item['size'] + . '
    seeders: ' + . $item['seeders'] + . ' | leechers: ' + . $item['leechers'] + . '
    info page'; + if(isset($item['title'])) + $this->items[] = $item; + } + } + } + } +} diff --git a/bridges/KernelBugTrackerBridge.php b/bridges/KernelBugTrackerBridge.php new file mode 100644 index 0000000..567ee50 --- /dev/null +++ b/bridges/KernelBugTrackerBridge.php @@ -0,0 +1,152 @@ + array( + 'id' => array( + 'name' => 'Bug tracking ID', + 'type' => 'number', + 'required' => true, + 'title' => 'Insert bug tracking ID', + 'exampleValue' => 121241 + ), + 'limit' => array( + 'name' => 'Number of comments to return', + 'type' => 'number', + 'required' => false, + 'title' => 'Specify number of comments to return', + 'defaultValue' => -1 + ), + 'sorting' => array( + 'name' => 'Sorting', + 'type' => 'list', + 'required' => false, + 'title' => 'Defines the sorting order of the comments returned', + 'defaultValue' => 'of', + 'values' => array( + 'Oldest first' => 'of', + 'Latest first' => 'lf' + ) + ) + ) + ); + + private $bugid = ''; + private $bugdesc = ''; + + public function collectData(){ + $limit = $this->getInput('limit'); + $sorting = $this->getInput('sorting'); + + // We use the print preview page for simplicity + $html = getSimpleHTMLDOMCached($this->getURI() . '&format=multiple', + 86400, + false, + null, + 0, + null, + true, + true, + DEFAULT_TARGET_CHARSET, + false, // Do NOT remove line breaks + DEFAULT_BR_TEXT, + DEFAULT_SPAN_TEXT); + + if($html === false) + returnServerError('Failed to load page!'); + + // Store header information into private members + $this->bugid = $html->find('#bugzilla-body', 0)->find('a', 0)->innertext; + $this->bugdesc = $html->find('table.bugfields', 0)->find('tr', 0)->find('td', 0)->innertext; + + // Get and limit comments + $comments = $html->find('div.bz_comment'); + + if($limit > 0 && count($comments) > $limit) { + $comments = array_slice($comments, count($comments) - $limit, $limit); + } + + // Order comments + switch($sorting) { + case 'lf': $comments = array_reverse($comments, true); + case 'of': + default: // Nothing to do, keep original order + } + + foreach($comments as $comment) { + $comment = $this->inlineStyles($comment); + + $item = array(); + $item['uri'] = $this->getURI() . '#' . $comment->id; + $item['author'] = $comment->find('span.bz_comment_user', 0)->innertext; + $item['title'] = $comment->find('span.bz_comment_number', 0)->find('a', 0)->innertext; + $item['timestamp'] = strtotime($comment->find('span.bz_comment_time', 0)->innertext); + $item['content'] = $comment->find('pre.bz_comment_text', 0)->innertext; + + // Fix line breaks (they use LF) + $item['content'] = str_replace("\n", '
    ', $item['content']); + + // Fix relative URIs + $item['content'] = $this->replaceRelativeURI($item['content']); + + $this->items[] = $item; + } + + } + + public function getURI(){ + switch($this->queriedContext) { + case 'Bug comments': + return parent::getURI() + . '/show_bug.cgi?id=' + . $this->getInput('id'); + break; + default: return parent::getURI(); + } + } + + public function getName(){ + switch($this->queriedContext) { + case 'Bug comments': + return 'Bug ' + . $this->bugid + . ' tracker for ' + . $this->bugdesc + . ' - ' + . parent::getName(); + break; + default: return parent::getName(); + } + } + + /** + * Replaces all relative URIs with absolute ones + * + * @param string $content The source string + * @return string Returns the source string with all relative URIs replaced + * by absolute ones. + */ + private function replaceRelativeURI($content){ + return preg_replace('/href="(?!http)/', 'href="' . self::URI . '/', $content); + } + + /** + * Adds styles as attributes to tags with known classes + * + * @param object $html A simplehtmldom object + * @return object Returns the original object with styles added as + * attributes. + */ + private function inlineStyles($html){ + foreach($html->find('.bz_obsolete') as $element) { + $element->style = 'text-decoration:line-through;'; + } + + return $html; + } + +} diff --git a/bridges/KonachanBridge.php b/bridges/KonachanBridge.php new file mode 100644 index 0000000..4250e8b --- /dev/null +++ b/bridges/KonachanBridge.php @@ -0,0 +1,11 @@ +find('p.itemText', 0)->innertext; + $item['content'] = utf8_encode($text); + + return $item; + } + + public function collectData(){ + $this->collectExpandableDatas('http://feeds.feedburner.com/Koreus-articles'); + } +} diff --git a/bridges/KununuBridge.php b/bridges/KununuBridge.php new file mode 100644 index 0000000..e99e135 --- /dev/null +++ b/bridges/KununuBridge.php @@ -0,0 +1,249 @@ + array( + 'site' => array( + 'name' => 'Site', + 'type' => 'list', + 'required' => true, + 'title' => 'Select your site', + 'values' => array( + 'Austria' => 'at', + 'Germany' => 'de', + 'Switzerland' => 'ch', + 'United States' => 'us' + ) + ), + 'full' => array( + 'name' => 'Load full article', + 'type' => 'checkbox', + 'required' => false, + 'exampleValue' => 'checked', + 'title' => 'Activate to load full article' + ) + ), + array( + 'company' => array( + 'name' => 'Company', + 'required' => true, + 'exampleValue' => 'kununu-us', + 'title' => 'Insert company name (i.e. Kununu US) or URI path (i.e. kununu-us)' + ) + ) + ); + + private $companyName = ''; + + public function getURI(){ + if(!is_null($this->getInput('company')) && !is_null($this->getInput('site'))) { + + $company = $this->fixCompanyName($this->getInput('company')); + $site = $this->getInput('site'); + $section = ''; + + switch($site) { + case 'at': + case 'de': + case 'ch': + $section = 'kommentare'; + break; + case 'us': + $section = 'reviews'; + break; + } + + return self::URI . $site . '/' . $company . '/' . $section; + } + + return parent::getURI(); + } + + function getName(){ + if(!is_null($this->getInput('company'))) { + $company = $this->fixCompanyName($this->getInput('company')); + return ($this->companyName ?: $company) . ' - ' . self::NAME; + } + + return parent::getName(); + } + + public function collectData(){ + $full = $this->getInput('full'); + + // Load page + $html = getSimpleHTMLDOMCached($this->getURI()); + if(!$html) + returnServerError('Unable to receive data from ' . $this->getURI() . '!'); + // Update name for this request + $this->companyName = $this->extractCompanyName($html); + + // Find the section with all the panels (reviews) + $section = $html->find('section.kununu-scroll-element', 0); + if($section === false) + returnServerError('Unable to find panel section!'); + + // Find all articles (within the panels) + $articles = $section->find('article'); + if($articles === false || empty($articles)) + returnServerError('Unable to find articles!'); + + // Go through all articles + foreach($articles as $article) { + $item = array(); + + $item['author'] = $this->extractArticleAuthorPosition($article); + $item['timestamp'] = $this->extractArticleDate($article); + $item['title'] = $this->extractArticleRating($article) + . ' : ' + . $this->extractArticleSummary($article); + + $item['uri'] = $this->extractArticleUri($article); + + if($full) + $item['content'] = $this->extractFullDescription($item['uri']); + else + $item['content'] = $this->extractArticleDescription($article); + + $this->items[] = $item; + } + } + + /** + * Fixes relative URLs in the given text + */ + private function fixUrl($text){ + return preg_replace('/href=(\'|\")\//i', 'href="'.self::URI, $text); + } + + /* + * Returns a fixed version of the provided company name + */ + private function fixCompanyName($company){ + $company = trim($company); + $company = str_replace(' ', '-', $company); + $company = strtolower($company); + return $this->encodeUmlauts($company); + } + + /** + * Encodes unmlauts in the given text + */ + private function encodeUmlauts($text){ + $umlauts = Array("/ä/","/ö/","/ü/","/Ä/","/Ö/","/Ü/","/ß/"); + $replace = Array("ae","oe","ue","Ae","Oe","Ue","ss"); + + return preg_replace($umlauts, $replace, $text); + } + + /** + * Returns the company name from the review html + */ + private function extractCompanyName($html){ + $company_name = $html->find('h1[itemprop=name]', 0); + if(is_null($company_name)) + returnServerError('Cannot find company name!'); + + return $company_name->plaintext; + } + + /** + * Returns the date from a given article + */ + private function extractArticleDate($article){ + // They conviniently provide a time attribute for us :) + $date = $article->find('meta[itemprop=dateCreated]', 0); + if(is_null($date)) + returnServerError('Cannot find article date!'); + + return strtotime($date->content); + } + + /** + * Returns the rating from a given article + */ + private function extractArticleRating($article){ + $rating = $article->find('span.rating', 0); + if(is_null($rating)) + returnServerError('Cannot find article rating!'); + + return $rating->getAttribute('aria-label'); + } + + /** + * Returns the summary from a given article + */ + private function extractArticleSummary($article){ + $summary = $article->find('[itemprop=name]', 0); + if(is_null($summary)) + returnServerError('Cannot find article summary!'); + + return strip_tags($summary->innertext); + } + + /** + * Returns the URI from a given article + */ + private function extractArticleUri($article){ + $anchor = $article->find('ku-company-review-more', 0); + if(is_null($anchor)) + returnServerError('Cannot find article URI!'); + + return self::URI . $anchor->{'review-url'}; + } + + /** + * Returns the position of the author from a given article + */ + private function extractArticleAuthorPosition($article){ + // We need to parse the user-content manually + $user_content = $article->find('div.user-content', 0); + if(is_null($user_content)) + returnServerError('Cannot find user content!'); + + // Go through all h2 elements to find index of required span (I know... it's stupid) + $author_position = 'Unknown'; + foreach($user_content->find('div') as $content) { + if(stristr(strtolower($content->plaintext), 'position')) { /* This works for at, ch, de, us */ + $author_position = $content->next_sibling()->plaintext; + break; + } + } + + return $author_position; + } + + /** + * Returns the description from a given article + */ + private function extractArticleDescription($article){ + $description = $article->find('[itemprop=reviewBody]', 0); + if(is_null($description)) + returnServerError('Cannot find article description!'); + + return $this->fixUrl($description->innertext); + } + + /** + * Returns the full description from a given uri + */ + private function extractFullDescription($uri){ + // Load full article + $html = getSimpleHTMLDOMCached($uri); + if($html === false) + returnServerError('Could not load full description!'); + + // Find the article + $article = $html->find('article', 0); + if(is_null($article)) + returnServerError('Cannot find article!'); + + // Luckily they use the same layout for the review overview and full article pages :) + return $this->extractArticleDescription($article); + } +} diff --git a/bridges/LWNprevBridge.php b/bridges/LWNprevBridge.php new file mode 100644 index 0000000..6d71c9d --- /dev/null +++ b/bridges/LWNprevBridge.php @@ -0,0 +1,265 @@ +nodeType === XML_TEXT_NODE) { + $nextNode = $node->nextSibling; + if(!$nextNode) { + break; + } + $node = $nextNode; + } + } + + private function jumpToPreviousTag(&$node){ + while($node && $node->nodeType === XML_TEXT_NODE) { + $previousNode = $node->previousSibling; + if(!$previousNode) { + break; + } + $node = $previousNode; + } + } + + public function collectData(){ + // Because the LWN page is written in loose HTML and not XHTML, + // Simple HTML Dom is not accurate enough for the job + $content = getContents($this->getURI()) + or returnServerError('No results for LWNprev'); + + $contents = explode('Page editor', $content); + + foreach($contents as $content) { + if(strpos($content, '') === false) { + $content = << +LWN{$content} +EOD; + } else { + $content = $content.''; + } + + libxml_use_internal_errors(true); + $html = new DOMDocument(); + $html->loadHTML($content); + libxml_clear_errors(); + + $edition = $html->getElementsByTagName('h1'); + if($edition->length !== 0) { + $text = $edition->item(0)->textContent; + $this->editionTimeStamp = strtotime( + substr($text, strpos($text, 'for ') + strlen('for ')) + ); + } + + if(strpos($content, 'Cat1HL') === false) { + $items = $this->getFeatureContents($html); + } elseif(strpos($content, 'Cat3HL') === false) { + $items = $this->getBriefItems($html); + } else { + $items = $this->getAnnouncements($html); + } + + $this->items = array_merge($this->items, $items); + } + } + + private function getArticleContent(&$title){ + $link = $title->firstChild; + $this->jumpToNextTag($link); + $item['uri'] = self::URI; + if($link->nodeName === 'a') { + $item['uri'] .= $link->getAttribute('href'); + } + + $item['timestamp'] = $this->editionTimeStamp; + + $node = $title; + $content = ''; + $contentEnd = false; + while(!$contentEnd) { + $node = $node->nextSibling; + if(!$node || ( + $node->nodeType !== XML_TEXT_NODE && + $node->nodeName === 'h2' || ( + !is_null($node->attributes) && + !is_null($class = $node->attributes->getNamedItem('class')) && + in_array($class->nodeValue, array('Cat1HL','Cat2HL')) + ) + ) + ) { + $contentEnd = true; + } else { + $content .= $node->C14N(); + } + } + $item['content'] = $content; + return $item; + } + + private function getFeatureContents(&$html){ + $items = array(); + foreach($html->getElementsByTagName('h2') as $title) { + if($title->getAttribute('class') !== 'SummaryHL') { + continue; + } + + $item = array(); + + $author = $title->nextSibling; + $this->jumpToNextTag($author); + if($author->getAttribute('class') === 'FeatureByline') { + $item['author'] = $author->getElementsByTagName('b')->item(0)->textContent; + } else { + continue; + } + + $item['title'] = $title->textContent; + + $items[] = array_merge($item, $this->getArticleContent($title)); + } + return $items; + } + + private function getItemPrefix(&$cat, &$cats){ + $cat1 = ''; + $cat2 = ''; + $cat3 = ''; + switch($cat->getAttribute('class')) { + case 'Cat3HL': + $cat3 = $cat->textContent; + $cat = $cat->previousSibling; + $this->jumpToPreviousTag($cat); + $cats[2] = $cat3; + if($cat->getAttribute('class') !== 'Cat2HL') { + break; + } + case 'Cat2HL': + $cat2 = $cat->textContent; + $cat = $cat->previousSibling; + $this->jumpToPreviousTag($cat); + $cats[1] = $cat2; + if(empty($cat3)) { + $cats[2] = ''; + } + if($cat->getAttribute('class') !== 'Cat1HL') { + break; + } + case 'Cat1HL': + $cat1 = $cat->textContent; + $cats[0] = $cat1; + if(empty($cat3)) { + $cats[2] = ''; + } + if(empty($cat2)) { + $cats[1] = ''; + } + break; + default: + break; + } + + $prefix = ''; + if(!empty($cats[0])) { + $prefix .= '['.$cats[0].($cats[1] ? '/'.$cats[1] : '').'] '; + } + return $prefix; + } + + private function getAnnouncements(&$html){ + $items = array(); + $cats = array('','',''); + + foreach($html->getElementsByTagName('p') as $newsletters) { + if($newsletters->getAttribute('class') !== 'Cat3HL') { + continue; + } + + $item = array(); + + $item['uri'] = self::URI.'#'.microtime(true); + + $item['timestamp'] = $this->editionTimeStamp;//+$URICounter; + + $item['author'] = 'LWN'; + + $cat = $newsletters->previousSibling; + $this->jumpToPreviousTag($cat); + $prefix = $this->getItemPrefix($cat, $cats); + $item['title'] = $prefix.' '.$newsletters->textContent; + + $node = $newsletters; + $content = ''; + $contentEnd = false; + while(!$contentEnd) { + $node = $node->nextSibling; + if(!$node || ( + $node->nodeType !== XML_TEXT_NODE && ( + !is_null($node->attributes) && + !is_null($class = $node->attributes->getNamedItem('class')) && + in_array($class->nodeValue, array('Cat1HL','Cat2HL','Cat3HL')) + ) + ) + ) { + $contentEnd = true; + } else { + $content .= $node->C14N(); + } + } + $item['content'] = $content; + $items[] = $item; + } + + foreach($html->getElementsByTagName('h2') as $title) { + if($title->getAttribute('class') !== 'SummaryHL') { + continue; + } + + $item = array(); + + $cat = $title->previousSibling; + $this->jumpToPreviousTag($cat); + $cat = $cat->previousSibling; + $this->jumpToPreviousTag($cat); + $prefix = $this->getItemPrefix($cat, $cats); + $item['title'] = $prefix.' '.$title->textContent; + $items[] = array_merge($item, $this->getArticleContent($title)); + } + + return $items; + } + + private function getBriefItems(&$html){ + $items = array(); + $cats = array('','',''); + foreach($html->getElementsByTagName('h2') as $title) { + if($title->getAttribute('class') !== 'SummaryHL') { + continue; + } + + $item = array(); + + $cat = $title->previousSibling; + $this->jumpToPreviousTag($cat); + $cat = $cat->previousSibling; + $this->jumpToPreviousTag($cat); + $prefix = $this->getItemPrefix($cat, $cats); + $item['title'] = $prefix.' '.$title->textContent; + $items[] = array_merge($item, $this->getArticleContent($title)); + } + + return $items; + } +} +?> diff --git a/bridges/LeBonCoinBridge.php b/bridges/LeBonCoinBridge.php new file mode 100755 index 0000000..d4da546 --- /dev/null +++ b/bridges/LeBonCoinBridge.php @@ -0,0 +1,190 @@ + array('name' => 'Mot Clé'), + 'r' => array( + 'name' => 'Région', + 'type' => 'list', + 'values' => array( + 'Toute la France' => 'ile_de_france/occasions', + 'Alsace' => 'alsace', + 'Aquitaine' => 'aquitaine', + 'Auvergne' => 'auvergne', + 'Basse Normandie' => 'basse_normandie', + 'Bourgogne' => 'bourgogne', + 'Bretagne' => 'bretagne', + 'Centre' => 'centre', + 'Champagne Ardenne' => 'champagne_ardenne', + 'Corse' => 'corse', + 'Franche Comté' => 'franche_comte', + 'Haute Normandie' => 'haute_normandie', + 'Ile de France' => 'ile_de_france', + 'Languedoc Roussillon' => 'languedoc_roussillon', + 'Limousin' => 'limousin', + 'Lorraine' => 'lorraine', + 'Midi Pyrénées' => 'midi_pyrenees', + 'Nord Pas De Calais' => 'nord_pas_de_calais', + 'Pays de la Loire' => 'pays_de_la_loire', + 'Picardie' => 'picardie', + 'Poitou Charentes' => 'poitou_charentes', + 'Provence Alpes Côte d\'Azur' => 'provence_alpes_cote_d_azur', + 'Rhône-Alpes' => 'rhone_alpes', + 'Guadeloupe' => 'guadeloupe', + 'Martinique' => 'martinique', + 'Guyane' => 'guyane', + 'Réunion' => 'reunion' + ) + ), + 'c' => array( + 'name' => 'Catégorie', + 'type' => 'list', + 'values' => array( + 'TOUS' => '', + 'EMPLOI' => '_emploi_', + 'VEHICULES' => array( + 'Tous' => '_vehicules_', + 'Voitures' => 'voitures', + 'Motos' => 'motos', + 'Caravaning' => 'caravaning', + 'Utilitaires' => 'utilitaires', + 'Équipement Auto' => 'equipement_auto', + 'Équipement Moto' => 'equipement_moto', + 'Équipement Caravaning' => 'equipement_caravaning', + 'Nautisme' => 'nautisme', + 'Équipement Nautisme' => 'equipement_nautisme' + ), + 'IMMOBILIER' => array( + 'Tous' => '_immobilier_', + 'Ventes immobilières' => 'ventes_immobilieres', + 'Locations' => 'locations', + 'Colocations' => 'colocations', + 'Bureaux & Commerces' => 'bureaux_commerces' + ), + 'VACANCES' => array( + 'Tous' => '_vacances_', + 'Location gîtes' => 'locations_gites', + 'Chambres d\'hôtes' => 'chambres_d_hotes', + 'Campings' => 'campings', + 'Hôtels' => 'hotels', + 'Hébergements insolites' => 'hebergements_insolites' + ), + 'MULTIMEDIA' => array( + 'Tous' => '_multimedia_', + 'Informatique' => 'informatique', + 'Consoles & Jeux vidéo' => 'consoles_jeux_video', + 'Image & Son' => 'image_son', + 'Téléphonie' => 'telephonie' + ), + 'LOISIRS' => array( + 'Tous' => '_loisirs_', + 'DVD / Films' => 'dvd_films', + 'CD / Musique' => 'cd_musique', + 'Livres' => 'livres', + 'Animaux' => 'animaux', + 'Vélos' => 'velos', + 'Sports & Hobbies' => 'sports_hobbies', + 'Instruments de musique' => 'instruments_de_musique', + 'Collection' => 'collection', + 'Jeux & Jouets' => 'jeux_jouets', + 'Vins & Gastronomie' => 'vins_gastronomie' + ), + 'MATÉRIEL PROFESSIONNEL' => array( + 'Tous' => '_materiel_professionnel_', + 'Matériel Agricole' => 'mateiel_agricole', + 'Transport - Manutention' => 'transport_manutention', + 'BTP - Chantier - Gros-œuvre' => 'btp_chantier_gros_oeuvre', + 'Outillage - Matériaux 2nd-œuvre' => 'outillage_materiaux_2nd_oeuvre', + 'Équipements Industriels' => 'equipement_industriels', + 'Restauration - Hôtellerie' => 'restauration_hotellerie', + 'Fournitures de Bureau' => 'fournitures_de_bureau', + 'Commerces & Marchés' => 'commerces_marches', + 'Matériel médical' => 'materiel_medical' + ), + 'SERVICES' => array( + 'Tous' => '_services_', + 'Prestations de services' => 'prestations_de_services', + 'Billetterie' => 'billetterie', + 'Évènements' => 'evenements', + 'Cours particuliers' => 'cours_particuliers', + 'Covoiturage' => 'covoiturage' + ), + 'MAISON' => array( + 'Tous' => '_maison_', + 'Ameublement' => 'ameublement', + 'Électroménager' => 'electromenager', + 'Arts de la table' => 'arts_de_la_table', + 'Décoration' => 'decoration', + 'Linge de maison' => 'linge_de_maison', + 'Bricolage' => 'bricolage', + 'Jardinage' => 'jardinage', + 'Vêtements' => 'vetements', + 'Chaussures' => 'chaussures', + 'Accessoires & Bagagerie' => 'accessoires_bagagerie', + 'Montres & Bijoux' => 'montres_bijoux', + 'Équipement bébé' => 'equipement_bebe', + 'Vêtements bébé' => 'vetements_bebe' + ), + 'AUTRES' => 'autres' + ) + ) + ) + ); + + public function collectData(){ + + $category = $this->getInput('c'); + if(empty($category)) { + $category = 'annonces'; + } + + $html = getSimpleHTMLDOM(self::URI + . $category + . '/offres/' + . $this->getInput('r') + . '/?f=a&th=1&q=' + . urlencode($this->getInput('k'))) + or returnServerError('Could not request LeBonCoin.'); + + $list = $html->find('.tabsContent', 0); + if($list === null) { + return; + } + + $tags = $list->find('li'); + + foreach($tags as $element) { + + $element = $element->find('a', 0); + + $item = array(); + $item['uri'] = $element->href; + $title = html_entity_decode($element->getAttribute('title')); + $content_image = $element->find('div.item_image', 0)->find('.lazyload', 0); + + if($content_image !== null) { + $content = 'thumbnail'; + } else { + $content = ""; + } + $date = $element->find('aside.item_absolute', 0)->find('p.item_sup', 0); + + $detailsList = $element->find('section.item_infos', 0); + + for($i = 0; $i <= 1; $i++) $content .= $detailsList->find('p.item_supp', $i)->plaintext; + $price = $detailsList->find('h3.item_price', 0); + $content .= $price === null ? '' : $price->plaintext; + + $item['title'] = $title; + $item['content'] = $content . $date; + $this->items[] = $item; + } + } +} diff --git a/bridges/LeMondeInformatiqueBridge.php b/bridges/LeMondeInformatiqueBridge.php new file mode 100644 index 0000000..706752f --- /dev/null +++ b/bridges/LeMondeInformatiqueBridge.php @@ -0,0 +1,44 @@ +collectExpandableDatas(self::URI . 'rss/rss.xml', 10); + } + + protected function parseItem($newsItem){ + $item = parent::parseItem($newsItem); + $article_html = getSimpleHTMLDOMCached($item['uri']) + or returnServerError('Could not request LeMondeInformatique: ' . $item['uri']); + $item['content'] = $this->cleanArticle($article_html->find('div#article', 0)->innertext); + $item['title'] = $article_html->find('h1.cleanprint-title', 0)->plaintext; + return $item; + } + + private function stripCDATA($string){ + $string = str_replace('', '', $string); + return $string; + } + + private function stripWithDelimiters($string, $start, $end){ + while(strpos($string, $start) !== false) { + $section_to_remove = substr($string, strpos($string, $start)); + $section_to_remove = substr($section_to_remove, 0, strpos($section_to_remove, $end) + strlen($end)); + $string = str_replace($section_to_remove, '', $string); + } + + return $string; + } + + private function cleanArticle($article_html){ + $article_html = $this->stripWithDelimiters($article_html, ''); + $article_html = $this->stripWithDelimiters($article_html, '

    '); + return $article_html; + } +} diff --git a/bridges/LegifranceJOBridge.php b/bridges/LegifranceJOBridge.php new file mode 100644 index 0000000..348be8f --- /dev/null +++ b/bridges/LegifranceJOBridge.php @@ -0,0 +1,68 @@ +author; + $item['timestamp'] = $this->timestamp; + $item['uri'] = $this->uri . '#' . count($this->items); + $item['title'] = $section->plaintext; + + if(!is_null($origin)) { + $item['title'] = '[ ' . $item['title'] . ' / ' . $subsection->plaintext . ' ] ' . $origin->plaintext; + $data = $origin; + } elseif(!is_null($subsection)) { + $item['title'] = '[ ' . $item['title'] . ' ] ' . $subsection->plaintext; + $data = $subsection; + } else { + $data = $section; + } + + $item['content'] = ''; + foreach($data->nextSibling()->find('a') as $content) { + $text = $content->plaintext; + $href = $content->nextSibling()->getAttribute('resource'); + $item['content'] .= '

    ' . $text . '

    '; + } + return $item; + } + + public function collectData(){ + $html = getSimpleHTMLDOM(self::URI) + or $this->returnServer('Unable to download ' . self::URI); + + $this->author = trim($html->find('h2.title', 0)->plaintext); + $uri = $html->find('h2.titleELI', 0)->plaintext; + $this->uri = trim(substr($uri, strpos($uri, 'https'))); + $this->timestamp = strtotime(substr($this->uri, strpos($this->uri, 'eli/jo/') + strlen('eli/jo/'))); + + foreach($html->find('h3') as $section) { + $subsections = $section->nextSibling()->find('h4'); + foreach($subsections as $subsection) { + $origins = $subsection->nextSibling()->find('h5'); + foreach($origins as $origin) { + $this->items[] = $this->extractItem($section, $subsection, $origin); + } + if(!empty($origins)) { + continue; + } + $this->items[] = $this->extractItem($section, $subsection); + } + if(!empty($subsections)) { + continue; + } + $this->items[] = $this->extractItem($section); + } + } +} diff --git a/bridges/LesJoiesDuCodeBridge.php b/bridges/LesJoiesDuCodeBridge.php new file mode 100644 index 0000000..34145a1 --- /dev/null +++ b/bridges/LesJoiesDuCodeBridge.php @@ -0,0 +1,45 @@ +find('div.blog-post') as $element) { + $item = array(); + $temp = $element->find('h1 a', 0); + $titre = html_entity_decode($temp->innertext); + $url = $temp->href; + + $temp = $element->find('div.blog-post-content', 0); + + // retrieve .gif instead of static .jpg + $images = $temp->find('p img'); + foreach($images as $image) { + $img_src = str_replace(".jpg", ".gif", $image->src); + $image->src = $img_src; + } + $content = $temp->innertext; + + $auteur = $temp->find('i', 0); + $pos = strpos($auteur->innertext, "by"); + + if($pos > 0) { + $auteur = trim(str_replace("*/", "", substr($auteur->innertext, ($pos + 2)))); + $item['author'] = $auteur; + } + + $item['content'] .= trim($content); + $item['uri'] = $url; + $item['title'] = trim($titre); + + $this->items[] = $item; + } + } +} diff --git a/bridges/LichessBridge.php b/bridges/LichessBridge.php new file mode 100644 index 0000000..bf7369f --- /dev/null +++ b/bridges/LichessBridge.php @@ -0,0 +1,31 @@ +collectExpandableDatas(self::URI . '.atom', 5); + } + + protected function parseItem($newsItem){ + $item = parent::parseItem($newsItem); + $item['content'] = $this->retrieveLichessPost($item['uri']); + return $item; + } + + private function retrieveLichessPost($blog_post_uri){ + $blog_post_html = getSimpleHTMLDOMCached($blog_post_uri); + $blog_post_div = $blog_post_html->find('#lichess_blog', 0); + + $post_chapo = $blog_post_div->find('.shortlede', 0)->innertext; + $post_content = $blog_post_div->find('.body', 0)->innertext; + + $content = '

    ' . $post_chapo . '

    '; + $content .= '
    ' . $post_content . '
    '; + + return $content; + } +} diff --git a/bridges/LinkedInCompanyBridge.php b/bridges/LinkedInCompanyBridge.php new file mode 100644 index 0000000..e629211 --- /dev/null +++ b/bridges/LinkedInCompanyBridge.php @@ -0,0 +1,37 @@ +apple)'; + + const PARAMETERS = array( array( + 'c' => array( + 'name' => 'Company name', + 'required' => true + ) + )); + + public function collectData(){ + $html = ''; + $link = self::URI . 'company/' . $this->getInput('c'); + + $html = getSimpleHTMLDOM($link) + or returnServerError('Could not request LinkedIn.'); + + foreach($html->find('//*[@id="my-feed-post"]/li') as $element) { + $title = $element->find('span.share-body', 0)->innertext; + if($title) { + $item = array(); + $item['uri'] = $link; + $item['title'] = mb_substr(strip_tags($element->find('span.share-body', 0)->innertext), 0, 100); + $item['content'] = strip_tags($element->find('span.share-body', 0)->innertext); + $this->items[] = $item; + $i++; + } + } + } +} diff --git a/bridges/LolibooruBridge.php b/bridges/LolibooruBridge.php new file mode 100644 index 0000000..b5bbd75 --- /dev/null +++ b/bridges/LolibooruBridge.php @@ -0,0 +1,11 @@ + array(), + 'Get popular mangas' => array( + 'category' => array( + 'name' => 'Category', + 'type' => 'list', + 'required' => true, + 'values' => array( + 'All' => 'all', + 'Action' => 'action', + 'Adventure' => 'adventure', + 'Comedy' => 'comedy', + 'Demons' => 'demons', + 'Drama' => 'drama', + 'Ecchi' => 'ecchi', + 'Fantasy' => 'fantasy', + 'Gender Bender' => 'gender-bender', + 'Harem' => 'harem', + 'Historical' => 'historical', + 'Horror' => 'horror', + 'Josei' => 'josei', + 'Magic' => 'magic', + 'Martial Arts' => 'martial-arts', + 'Mature' => 'mature', + 'Mecha' => 'mecha', + 'Military' => 'military', + 'Mystery' => 'mystery', + 'One Shot' => 'one-shot', + 'Psychological' => 'psychological', + 'Romance' => 'romance', + 'School Life' => 'school-life', + 'Sci-Fi' => 'sci-fi', + 'Seinen' => 'seinen', + 'Shoujo' => 'shoujo', + 'Shoujoai' => 'shoujoai', + 'Shounen' => 'shounen', + 'Shounenai' => 'shounenai', + 'Slice of Life' => 'slice-of-life', + 'Smut' => 'smut', + 'Sports' => 'sports', + 'Super Power' => 'super-power', + 'Supernatural' => 'supernatural', + 'Tragedy' => 'tragedy', + 'Vampire' => 'vampire', + 'Yaoi' => 'yaoi', + 'Yuri' => 'yuri' + ), + 'exampleValue' => 'All', + 'title' => 'Select your category' + ) + ), + 'Get manga updates' => array( + 'path' => array( + 'name' => 'Path', + 'required' => true, + 'pattern' => '[a-zA-Z0-9-_]*', + 'exampleValue' => 'bleach, umi-no-kishidan', + 'title' => 'URL part of desired manga' + ), + 'limit' => array( + 'name' => 'Limit', + 'type' => 'number', + 'defaultValue' => 10, + 'title' => 'Number of items to return [-1 returns all]' + ) + ) + ); + + private $request = ''; + + public function collectData(){ + // We'll use the DOM parser for this as it makes navigation easier + $html = getContents($this->getURI()); + if(!$html) { + returnClientError('Could not receive data for ' . $path . '!'); + } + libxml_use_internal_errors(true); + $doc = new DomDocument; + @$doc->loadHTML($html); + libxml_clear_errors(); + + // Navigate via XPath + $xpath = new DomXPath($doc); + + $this->request = ''; + switch($this->queriedContext) { + case 'Get latest updates': + $this->request = 'Latest updates'; + $this->getLatestUpdates($xpath); + break; + case 'Get popular mangas': + // Find manga name within "Popular mangas for ..." + $pagetitle = $xpath->query(".//*[@id='bodyalt']/h1")->item(0)->nodeValue; + $this->request = substr($pagetitle, 0, strrpos($pagetitle, " -")); + $this->getPopularMangas($xpath); + break; + case 'Get manga updates': + $limit = $this->getInput('limit'); + if(empty($limit)) { + $limit = self::PARAMETERS[$this->queriedContext]['limit']['defaultValue']; + } + + $this->request = $xpath->query(".//*[@id='mangaproperties']//*[@class='aname']") + ->item(0) + ->nodeValue; + + $this->getMangaUpdates($xpath, $limit); + break; + } + + // Return some dummy-data if no content available + if(empty($this->items)) { + $item = array(); + $item['content'] = "

    No updates available

    "; + + $this->items[] = $item; + } + } + + private function getLatestUpdates($xpath){ + // Query each item (consists of Manga + chapters) + $nodes = $xpath->query("//*[@id='latestchapters']/table//td"); + + foreach ($nodes as $node) { + // Query the manga + $manga = $xpath->query("a[@class='chapter']", $node)->item(0); + + // Collect the chapters for each Manga + $chapters = $xpath->query("a[@class='chaptersrec']", $node); + + if (isset($manga) && $chapters->length >= 1) { + $item = array(); + $item['uri'] = self::URI . htmlspecialchars($manga->getAttribute('href')); + $item['title'] = htmlspecialchars($manga->nodeValue); + + // Add each chapter to the feed + $item['content'] = ""; + + foreach ($chapters as $chapter) { + if($item['content'] <> "") { + $item['content'] .= "
    "; + } + $item['content'] .= "" + . htmlspecialchars($chapter->nodeValue) + . ""; + } + + $this->items[] = $item; + } + } + } + + private function getPopularMangas($xpath){ + // Query all mangas + $mangas = $xpath->query("//*[@id='mangaresults']/*[@class='mangaresultitem']"); + + foreach ($mangas as $manga) { + + // The thumbnail is encrypted in a css-style... + // format: "background-image:url('')" + $mangaimgelement = $xpath->query(".//*[@class='imgsearchresults']", $manga) + ->item(0) + ->getAttribute('style'); + $thumbnail = substr($mangaimgelement, 22, strlen($mangaimgelement) - 24); + + $item = array(); + $item['title'] = htmlspecialchars($xpath->query(".//*[@class='manga_name']//a", $manga) + ->item(0) + ->nodeValue); + $item['uri'] = self::URI . $xpath->query(".//*[@class='manga_name']//a", $manga) + ->item(0) + ->getAttribute('href'); + $item['author'] = htmlspecialchars($xpath->query("//*[@class='author_name']", $manga) + ->item(0) + ->nodeValue); + $item['chaptercount'] = $xpath->query(".//*[@class='chapter_count']", $manga) + ->item(0) + ->nodeValue; + $item['genre'] = htmlspecialchars($xpath->query(".//*[@class='manga_genre']", $manga) + ->item(0) + ->nodeValue); + $item['content'] = <<{$item['title']} +

    {$item['genre']}

    +

    {$item['chaptercount']}

    +EOD; + $this->items[] = $item; + } + } + + private function getMangaUpdates($xpath, $limit){ + $query = "(.//*[@id='listing']//tr)[position() > 1]"; + + if($limit !== -1) { + $query = "(.//*[@id='listing']//tr)[position() > 1][position() > last() - {$limit}]"; + } + + $chapters = $xpath->query($query); + + foreach ($chapters as $chapter) { + $item = array(); + $item['title'] = htmlspecialchars($xpath->query("td[1]", $chapter) + ->item(0) + ->nodeValue); + $item['uri'] = self::URI . $xpath->query("td[1]/a", $chapter) + ->item(0) + ->getAttribute('href'); + $item['timestamp'] = strtotime($xpath->query("td[2]", $chapter) + ->item(0) + ->nodeValue); + array_unshift($this->items, $item); + } + } + + public function getURI(){ + switch($this->queriedContext) { + case 'Get latest updates': + $path = "latest"; + break; + case 'Get popular mangas': + $path = "popular"; + if($this->getInput('category') !== "all") { + $path .= "/" . $this->getInput('category'); + } + break; + case 'Get manga updates': + $path = $this->getInput('path'); + break; + default: return parent::getURI(); + } + return self::URI . '/' . $path; + } + + public function getName(){ + return (!empty($this->request) ? $this->request . ' - ' : '') . 'Mangareader Bridge'; + } +} diff --git a/bridges/MilbooruBridge.php b/bridges/MilbooruBridge.php new file mode 100644 index 0000000..c3b633e --- /dev/null +++ b/bridges/MilbooruBridge.php @@ -0,0 +1,11 @@ + array( + 'name' => 'username', + 'required' => true, + ) + )); + + public function getName(){ + if(!is_null($this->getInput('u'))) { + return 'MixCloud - ' . $this->getInput('u'); + } + + return parent::getName(); + } + + public function collectData(){ + + $html = getSimpleHTMLDOM(self::URI . $this->getInput('u')) + or returnServerError('Could not request MixCloud.'); + + foreach($html->find('section.card') as $element) { + + $item = array(); + + $item['uri'] = self::URI . $element->find('hgroup.card-title h1 a', 0)->getAttribute('href'); + $item['title'] = html_entity_decode( + $element->find('hgroup.card-title h1 a span', 0)->getAttribute('title'), + ENT_QUOTES + ); + + $image = $element->find('a.album-art img', 0); + + if($image) { + $item['content'] = ''; + } + + $item['author'] = trim($element->find('hgroup.card-title h2 a', 0)->innertext); + + $this->items[] = $item; + } + } +} diff --git a/bridges/MoebooruBridge.php b/bridges/MoebooruBridge.php new file mode 100644 index 0000000..9d9a625 --- /dev/null +++ b/bridges/MoebooruBridge.php @@ -0,0 +1,56 @@ + array( + 'name' => 'page', + 'defaultValue' => 1, + 'type' => 'number' + ), + 't' => array( + 'name' => 'tags' + ) + )); + + protected function getFullURI(){ + return $this->getURI() + . 'post?page=' + . $this->getInput('p') + . '&tags=' + . urlencode($this->getInput('t')); + } + + public function collectData(){ + $html = getSimpleHTMLDOM($this->getFullURI()) + or returnServerError('Could not request ' . $this->getName()); + + $input_json = explode('Post.register(', $html); + foreach($input_json as $element) + $data[] = preg_replace('/}\)(.*)/', '}', $element); + unset($data[0]); + + foreach($data as $datai) { + $json = json_decode($datai, true); + $item = array(); + $item['uri'] = $this->getURI() . '/post/show/' . $json['id']; + $item['postid'] = $json['id']; + $item['timestamp'] = $json['created_at']; + $item['imageUri'] = $json['file_url']; + $item['title'] = $this->getName() . ' | ' . $json['id']; + $item['content'] = '
    Tags: ' + . $json['tags']; + + $this->items[] = $item; + } + } +} diff --git a/bridges/MoinMoinBridge.php b/bridges/MoinMoinBridge.php new file mode 100644 index 0000000..5b41924 --- /dev/null +++ b/bridges/MoinMoinBridge.php @@ -0,0 +1,327 @@ + array( + 'name' => 'Source', + 'type' => 'text', + 'required' => true, + 'title' => 'Insert wiki page URI (e.g.: https://moinmo.in/MoinMoin)', + 'exampleValue' => 'https://moinmo.in/MoinMoin' + ), + 'separator' => array( + 'name' => 'Separator', + 'type' => 'list', + 'requied' => true, + 'title' => 'Defines the separtor for splitting content into feeds', + 'defaultValue' => 'h2', + 'values' => array( + 'Header (h1)' => 'h1', + 'Header (h2)' => 'h2', + 'Header (h3)' => 'h3', + 'List element (li)' => 'li', + 'Anchor (a)' => 'a' + ) + ), + 'limit' => array( + 'name' => 'Limit', + 'type' => 'number', + 'required' => false, + 'title' => 'Number of items to return (from top)', + 'defaultValue' => -1 + ), + 'content' => array( + 'name' => 'Content', + 'type' => 'list', + 'required' => false, + 'title' => 'Defines how feed contents are build', + 'defaultValue' => 'separator', + 'values' => array( + 'By separator' => 'separator', + 'Follow link (only for anchor)' => 'follow', + 'None' => 'none' + ) + ) + ) + ); + + private $title = ''; + + public function collectData(){ + /* MoinMoin uses a rather unpleasent representation of HTML. Instead of + * using tags like
    , ,
    , etc... it uses + *
    , and

    . Also each line is literaly identified via + * IDs. The only way to distinguish content is via headers, though not + * in all cases. + * + * Example (indented for the sake of readability): + * ... + * + * + * + * + * + * + * + * + * + *

    MoinMoin is a Wiki software implemented in + * Python + * and distributed as Free Software under + * GNU GPL license. + * ... + */ + $html = getSimpleHTMLDOM($this->getInput('source')) + or returnServerError('Could not load ' . $this->getInput('source')); + + // Some anchors link to local sites or local IDs (both don't work well + // in feeds) + $html = $this->fixAnchors($html); + + $this->title = $html->find('title', 0)->innertext . ' | ' . self::NAME; + + // Here we focus on simple author and timestamp information from the given + // page. Later we update this information in case the anchor is followed. + $author = $this->findAuthor($html); + $timestamp = $this->findTimestamp($html); + + $sections = $this->splitSections($html); + + foreach($sections as $section) { + $item = array(); + + $item['uri'] = $this->findSectionAnchor($section[0]); + + switch($this->getInput('content')) { + case 'none': // Do not return any content + break; + case 'follow': // Follow the anchor + // We can only follow anchors (use default otherwise) + if($this->getInput('separator') === 'a') { + $content = $this->followAnchor($item['uri']); + + // Return only actual content + $item['content'] = $content->find('div#page', 0)->innertext; + + // Each page could have its own author and timestamp + $author = $this->findAuthor($content); + $timestamp = $this->findTimestamp($content); + + break; + } + case 'separator': + default: // Use contents from the current page + $item['content'] = $this->cleanArticle($section[2]); + } + + if(!is_null($author)) $item['author'] = $author; + if(!is_null($timestamp)) $item['timestamp'] = $timestamp; + $item['title'] = strip_tags($section[1]); + + // Skip items with empty title + if(empty(trim($item['title']))) { + continue; + } + + $this->items[] = $item; + + if($this->getInput('limit') > 0 + && count($this->items) >= $this->getInput('limit')) { + break; + } + } + } + + public function getName(){ + return $this->title ?: parent::getName(); + } + + public function getURI(){ + return $this->getInput('source') ?: parent::getURI(); + } + + /** + * Splits the html into sections. + * + * Returns an array with one element per section. Each element consists of: + * [0] The entire section + * [1] The section title + * [2] The section content + */ + private function splitSections($html){ + $content = $html->find('div#page', 0)->innertext + or returnServerError('Unable to find

    !'); + + $sections = array(); + + $regex = implode( + '', + array( + "\<{$this->getInput('separator')}.+?(?=\>)\>", + "(.+?)(?=\<\/{$this->getInput('separator')}\>)", + "\<\/{$this->getInput('separator')}\>", + "(.+?)((?=\<{$this->getInput('separator')})|(?=\find('title', 0)->innertext, + $content + ) + ); + } + + return $sections; + } + + /** + * Returns the anchor for a given section + */ + private function findSectionAnchor($section){ + $html = str_get_html($section); + + // For IDs + $anchor = $html->find($this->getInput('separator') . '[id=]', 0); + if(!is_null($anchor)) { + return $this->getInput('source') . '#' . $anchor->id; + } + + // For actual anchors + $anchor = $html->find($this->getInput('separator') . '[href=]', 0); + if(!is_null($anchor)) { + return $anchor->href; + } + + // Nothing found + return $this->getInput('source'); + } + + /** + * Returns the author + * + * Notice: Some pages don't provide author information + */ + private function findAuthor($html){ + /* Example: + *

    MoinMoin: LocalSpellingWords + * (last edited 2017-02-16 15:36:31 by hosted-by)

    + */ + $pageinfo = $html->find('[id="pageinfo"]', 0); + + if(is_null($pageinfo)) { + return null; + } else { + $author = $pageinfo->find('[title=]', 0); + if(is_null($author)) { + return null; + } else { + return trim(explode('@', $author->title)[0]); + } + } + } + + /** + * Returns the time of last edit + * + * Notice: Some pages don't provide this information + */ + private function findTimestamp($html){ + // See example of findAuthor() + $pageinfo = $html->find('[id="pageinfo"]', 0); + + if(is_null($pageinfo)) { + return null; + } else { + $timestamp = $pageinfo->innertext; + $matches = array(); + preg_match('/.+?(?=\().+?(?=\d)([0-9\-\s\:]+)/m', $pageinfo, $matches); + return strtotime($matches[1]); + } + } + + /** + * Returns the original HTML with all anchors fixed (makes relative anchors + * absolute) + */ + private function fixAnchors($html, $source = null){ + + $source = $source ?: $this->getURI(); + + foreach($html->find('a') as $anchor) { + switch(substr($anchor->href, 0, 1)) { + case 'h': // http or https, no actions required + break; + case '/': // some relative path + $anchor->href = $this->findDomain($source) . $anchor->href; + break; + case '#': // it's an ID + default: // probably something like ? or &, skip empty ones + if(!isset($anchor->href)) + break; + $anchor->href = $source . $anchor->href; + } + } + + return $html; + } + + /** + * Loads the full article of a given anchor (if the anchor is from the same + * wiki domain) + */ + private function followAnchor($anchor){ + if(strrpos($anchor, $this->findDomain($this->getInput('source')) === false)) { + return null; + } + + $html = getSimpleHTMLDOMCached($anchor); + if(!$html) { // Cannot load article + return null; + } + + return $this->fixAnchors($html, $anchor); + } + + /** + * Finds the domain for a given URI + */ + private function findDomain($uri){ + $matches = array(); + preg_match('/(http[s]{0,1}:\/\/.+?(?=\/))/', $uri, $matches); + return $matches[1]; + } + + /* This function is a copy from CNETBridge */ + private function stripWithDelimiters($string, $start, $end){ + while(strpos($string, $start) !== false) { + $section_to_remove = substr($string, strpos($string, $start)); + $section_to_remove = substr($section_to_remove, 0, strpos($section_to_remove, $end) + strlen($end)); + $string = str_replace($section_to_remove, '', $string); + } + + return $string; + } + + /* This function is based on CNETBridge */ + private function cleanArticle($article_html){ + $article_html = $this->stripWithDelimiters($article_html, ''); + return $article_html; + } +} diff --git a/bridges/MondeDiploBridge.php b/bridges/MondeDiploBridge.php new file mode 100644 index 0000000..85f771e --- /dev/null +++ b/bridges/MondeDiploBridge.php @@ -0,0 +1,26 @@ +find('div.unarticle') as $article) { + $element = $article->parent(); + $item = array(); + $item['uri'] = self::URI . $element->href; + $item['title'] = $element->find('h3', 0)->plaintext; + $item['content'] = $element->find('div.dates_auteurs', 0)->plaintext + . '
    ' + . strstr($element->find('div', 0)->plaintext, $element->find('div.dates_auteurs', 0)->plaintext, true); + + $this->items[] = $item; + } + } +} diff --git a/bridges/MsnMondeBridge.php b/bridges/MsnMondeBridge.php new file mode 100644 index 0000000..12d3d2f --- /dev/null +++ b/bridges/MsnMondeBridge.php @@ -0,0 +1,35 @@ +find('#content', 0)->find('article', 0)->find('section', 0)->plaintext; + $item['timestamp'] = strtotime($html2->find('.authorinfo-txt', 0)->find('time', 0)->datetime); + } + + public function collectData(){ + $html = getSimpleHTMLDOM($this->getURI()) + or returnServerError('Could not request MsnMonde.'); + + $limit = 0; + foreach($html->find('.smalla') as $article) { + if($limit < 10) { + $item = array(); + $item['title'] = utf8_decode($article->find('h4', 0)->innertext); + $item['uri'] = self::URI . utf8_decode($article->find('a', 0)->href); + $this->msnMondeExtractContent($item['uri'], $item); + $this->items[] = $item; + $limit++; + } + } + } +} diff --git a/bridges/MspabooruBridge.php b/bridges/MspabooruBridge.php new file mode 100644 index 0000000..00a7bd7 --- /dev/null +++ b/bridges/MspabooruBridge.php @@ -0,0 +1,12 @@ +", $html->find('b', 0)->innertext); + + for($i = 0; $i < 3; $i++) { + $line = $list[$i]; + $item = array(); + + $uri_page = $html->find('a', $i + 3)->href; + $uri = self::URI . $uri_page; + $item['uri'] = $uri; + + $picture_html = getSimpleHTMLDOM($uri); + $picture_html_string = $picture_html->innertext; + + //Extract image and explanation + $media = $picture_html->find('p', 1)->innertext; + $media = strstr($media, '
    '); + $media = preg_replace('/
    /', '', $media, 1); + $explanation = $picture_html->find('p', 2)->innertext; + + //Extract date from the picture page + $date = explode(" ", $picture_html->find('p', 1)->innertext); + $item['timestamp'] = strtotime($date[4] . $date[3] . $date[2]); + + //Other informations + $item['content'] = $media . '
    ' . $explanation; + $item['title'] = $picture_html->find('b', 0)->innertext; + $this->items[] = $item; + } + } +} diff --git a/bridges/NeuviemeArtBridge.php b/bridges/NeuviemeArtBridge.php new file mode 100644 index 0000000..d0954fc --- /dev/null +++ b/bridges/NeuviemeArtBridge.php @@ -0,0 +1,57 @@ +find('img.img_full') as $img) { + if ($img->alt == $item['title']) { + $article_image = self::URI . $img->src; + break; + } + } + + $article_content = ''; + if($article_image) { + $article_content = '

    '; + } + $article_content .= str_replace( + 'src="/', 'src="' . self::URI, + $article_html->find('div.newsGenerique_con', 0)->innertext + ); + $article_content = $this->stripWithDelimiters($article_content, ''); + $article_content = $this->stripWithDelimiters($article_content, ''); + $article_content = $this->stripWithDelimiters($article_content, ''); + + $item['content'] = $article_content; + + return $item; + } + + public function collectData(){ + $feedUrl = self::URI . '9emeart.rss'; + $this->collectExpandableDatas($feedUrl); + } +} diff --git a/bridges/NextInpactBridge.php b/bridges/NextInpactBridge.php new file mode 100644 index 0000000..5de5c8b --- /dev/null +++ b/bridges/NextInpactBridge.php @@ -0,0 +1,34 @@ +collectExpandableDatas(self::URI . 'rss/news.xml', 10); + } + + protected function parseItem($newsItem){ + $item = parent::parseItem($newsItem); + $item['content'] = $this->extractContent($item['uri']); + return $item; + } + + private function extractContent($url){ + $html2 = getSimpleHTMLDOMCached($url); + $text = '

    ' + . $html2->find('span.sub_title', 0)->innertext + . '

    -

    ' + . $html2->find('div[itemprop=articleBody]', 0)->innertext + . '
    '; + + $premium_article = $html2->find('h2.title_reserve_article', 0); + if (is_object($premium_article)) + $text = $text . '

    ' . $premium_article->innertext . '

    '; + return $text; + } +} diff --git a/bridges/NextgovBridge.php b/bridges/NextgovBridge.php new file mode 100644 index 0000000..370b0bf --- /dev/null +++ b/bridges/NextgovBridge.php @@ -0,0 +1,74 @@ + array( + 'name' => 'Category', + 'type' => 'list', + 'values' => array( + 'All' => 'all', + 'Technology News' => 'technology-news', + 'CIO Briefing' => 'cio-briefing', + 'Emerging Tech' => 'emerging-tech', + 'Cloud' => 'cloud-computing', + 'Cybersecurity' => 'cybersecurity', + 'Mobile' => 'mobile', + 'Health' => 'health', + 'Defense' => 'defense', + 'Big Data' => 'big-data' + ) + ) + )); + + public function collectData(){ + $this->collectExpandableDatas(self::URI . 'rss/' . $this->getInput('category') . '/', 10); + } + + protected function parseItem($newsItem){ + $item = parent::parseItem($newsItem); + + $item['content'] = ''; + + $namespaces = $newsItem->getNamespaces(true); + if(isset($namespaces['media'])) { + $media = $newsItem->children($namespaces['media']); + if(isset($media->content)) { + $attributes = $media->content->attributes(); + $item['content'] = ''; + } + } + + $item['content'] .= $this->extractContent($item['uri']); + return $item; + } + + private function stripWithDelimiters($string, $start, $end){ + while (strpos($string, $start) !== false) { + $section_to_remove = substr($string, strpos($string, $start)); + $section_to_remove = substr($section_to_remove, 0, strpos($section_to_remove, $end) + strlen($end)); + $string = str_replace($section_to_remove, '', $string); + } + + return $string; + } + + private function extractContent($url){ + $article = getSimpleHTMLDOMCached($url) + or returnServerError('Could not request Nextgov: ' . $url); + + $contents = $article->find('div.wysiwyg', 0)->innertext; + $contents = $this->stripWithDelimiters($contents, '
    ', '
    '); + $contents = $this->stripWithDelimiters($contents, ''); //ad outer div + return $this->stripWithDelimiters($contents, ''); + $contents = ($article_thumbnail == '' ? '' : '

    ') + . '

    ' + . $article_subtitle + . '

    ' + . trim($contents); + } +} diff --git a/bridges/NiceMatinBridge.php b/bridges/NiceMatinBridge.php new file mode 100644 index 0000000..117c779 --- /dev/null +++ b/bridges/NiceMatinBridge.php @@ -0,0 +1,32 @@ +collectExpandableDatas(self::URI . 'derniere-minute/rss', 10); + } + + protected function parseItem($newsItem){ + $item = parent::parseItem($newsItem); + $item['content'] = $this->extractContent($item['uri']); + return $item; + } + + private function extractContent($url){ + $html = getSimpleHTMLDOMCached($url); + if(!$html) + return 'Could not acquire content from url: ' . $url . '!'; + + $content = $html->find('article', 0); + if(!$content) + return 'Could not find \'section\'!'; + + $text = preg_replace('#(.*?)#is', '', $content->innertext); + $text = strip_tags($text, '

    '); + return $text; + } +} diff --git a/bridges/NovelUpdatesBridge.php b/bridges/NovelUpdatesBridge.php new file mode 100644 index 0000000..729eb48 --- /dev/null +++ b/bridges/NovelUpdatesBridge.php @@ -0,0 +1,69 @@ + array( + 'name' => 'Novel name as found in the url', + 'exampleValue' => 'spirit-realm', + 'required' => true + ) + )); + + private $seriesTitle = ''; + + public function getURI(){ + if(!is_null($this->getInput('n'))) { + return static::URI . '/series/' . $this->getInput('n') . '/'; + } + + return parent::getURI(); + } + + public function collectData(){ + $fullhtml = getSimpleHTMLDOM($this->getURI()) + or returnServerError('Could not request NovelUpdates, novel "' . $this->getInput('n') . '" not found'); + + $this->seriesTitle = $fullhtml->find('h4.seriestitle', 0)->plaintext; + // dirty fix for nasty simpledom bug: https://github.com/sebsauvage/rss-bridge/issues/259 + // forcefully removes tbody + $html = $fullhtml->find('table#myTable', 0)->innertext; + $html = stristr($html, '

    '); //strip thead + $html = stristr($html, ''); //remove tbody + $html = str_get_html(stristr($html, '', true)); //remove last tbody and get back as an array + foreach($html->find('tr') as $element) { + $item = array(); + $item['uri'] = $element->find('td', 2)->find('a', 0)->href; + $item['title'] = $element->find('td', 2)->find('a', 0)->plaintext; + $item['team'] = $element->find('td', 1)->innertext; + $item['timestamp'] = strtotime($element->find('td', 0)->plaintext); + $item['content'] = '' + . $this->seriesTitle + . ' - ' + . $item['title'] + . ' by ' + . $item['team'] + . '
    ' + . $fullhtml->find('div.seriesimg', 0)->innertext + . ''; + + $this->items[] = $item; + } + } + + public function getName(){ + if(!empty($this->seriesTitle)) { + return $this->seriesTitle . ' - ' . static::NAME; + } + + return parent::getName(); + } +} diff --git a/bridges/OpenClassroomsBridge.php b/bridges/OpenClassroomsBridge.php new file mode 100644 index 0000000..5f0daca --- /dev/null +++ b/bridges/OpenClassroomsBridge.php @@ -0,0 +1,49 @@ + array( + 'name' => 'Catégorie', + 'type' => 'list', + 'required' => true, + 'values' => array( + 'Arts & Culture' => 'arts', + 'Code' => 'code', + 'Design' => 'design', + 'Entreprise' => 'business', + 'Numérique' => 'digital', + 'Sciences' => 'sciences', + 'Sciences Humaines' => 'humainities', + 'Systèmes d\'information' => 'it', + 'Autres' => 'others' + ) + ) + )); + + public function getURI(){ + if(!is_null($this->getInput('u'))) { + return self::URI . '/courses?categories=' . $this->getInput('u') . '&title=&sort=updatedAt+desc'; + } + + return parent::getURI(); + } + + public function collectData(){ + $html = getSimpleHTMLDOM($this->getURI()) + or returnServerError('Could not request OpenClassrooms.'); + + foreach($html->find('.courseListItem') as $element) { + $item = array(); + $item['uri'] = self::URI . $element->find('a', 0)->href; + $item['title'] = $element->find('h3', 0)->plaintext; + $item['content'] = $element->find('slidingItem__descriptionContent', 0)->plaintext; + $this->items[] = $item; + } + } +} diff --git a/bridges/ParuVenduImmoBridge.php b/bridges/ParuVenduImmoBridge.php new file mode 100644 index 0000000..a2e2b33 --- /dev/null +++ b/bridges/ParuVenduImmoBridge.php @@ -0,0 +1,102 @@ + array( + 'name' => 'Minimal surface m²', + 'type' => 'number' + ), + 'maxprice' => array( + 'name' => 'Max price', + 'type' => 'number' + ), + 'pa' => array( + 'name' => 'Country code', + 'exampleValue' => 'FR' + ), + 'lo' => array( + 'name' => 'department numbers or postal codes, comma-separated' + ) + )); + + public function collectData(){ + $html = getSimpleHTMLDOM($this->getURI()) + or returnServerError('Could not request paruvendu.'); + + foreach($html->find('div.annonce a') as $element) { + + if(!$element->title) { + continue; + } + + $img = ''; + foreach($element->find('span.img img') as $img) { + if($img->original) { + $img = ''; + } + } + + $desc = $element->find('span.desc')[0]->innertext; + $desc = str_replace("voir l'annonce", '', $desc); + + $price = $element->find('span.price')[0]->innertext; + + list($href) = explode('#', $element->href); + + $item = array(); + $item['uri'] = self::URI . $href; + $item['title'] = $element->title; + $item['content'] = $img . $desc . $price; + $this->items[] = $item; + } + } + + public function getURI(){ + $appartment = '&tbApp=1&tbDup=1&tbChb=1&tbLof=1&tbAtl=1&tbPla=1'; + $maison = '&tbMai=1&tbVil=1&tbCha=1&tbPro=1&tbHot=1&tbMou=1&tbFer=1'; + $link = self::URI + . '/immobilier/annonceimmofo/liste/listeAnnonces?tt=1' + . $appartment + . $maison; + + if($this->getInput('minarea')) { + $link .= '&sur0=' . urlencode($this->getInput('minarea')); + } + + if($this->getInput('maxprice')) { + $link .= '&px1=' . urlencode($this->getInput('maxprice')); + } + + if($this->getInput('pa')) { + $link .= '&pa=' . urlencode($this->getInput('pa')); + } + + if($this->getInput('lo')) { + $link .= '&lo=' . urlencode($this->getInput('lo')); + } + return $link; + } + + public function getName(){ + if(!is_null($this->getInput('minarea'))) { + $request = ''; + $minarea = $this->getInput('minarea'); + if(!empty($minarea)) { + $request .= ' ' . $minarea . ' m2'; + } + $location = $this->getInput('lo'); + if(!empty($location)) { + $request .= ' In: ' . $location; + } + return 'Paru Vendu Immobilier' . $request; + } + + return parent::getName(); + } +} diff --git a/bridges/PickyWallpapersBridge.php b/bridges/PickyWallpapersBridge.php new file mode 100644 index 0000000..6c26df7 --- /dev/null +++ b/bridges/PickyWallpapersBridge.php @@ -0,0 +1,101 @@ + array( + 'name' => 'category', + 'required' => true + ), + 's' => array( + 'name' => 'subcategory' + ), + 'm' => array( + 'name' => 'Max number of wallpapers', + 'defaultValue' => 12, + 'type' => 'number' + ), + 'r' => array( + 'name' => 'resolution', + 'exampleValue' => '1920x1200, 1680x1050,…', + 'defaultValue' => '1920x1200', + 'pattern' => '[0-9]{3,4}x[0-9]{3,4}' + ) + )); + + public function collectData(){ + $lastpage = 1; + $num = 0; + $max = $this->getInput('m'); + $resolution = $this->getInput('r'); // Wide wallpaper default + + for($page = 1; $page <= $lastpage; $page++) { + $html = getSimpleHTMLDOM($this->getURI() . '/page-' . $page . '/') + or returnServerError('No results for this query.'); + + if($page === 1) { + preg_match('/page-(\d+)\/$/', $html->find('.pages li a', -2)->href, $matches); + $lastpage = min($matches[1], ceil($max / 12)); + } + + foreach($html->find('.items li img') as $element) { + $item = array(); + $item['uri'] = str_replace('www', 'wallpaper', self::URI) + . '/' + . $resolution + . '/' + . basename($element->src); + + $item['timestamp'] = time(); + $item['title'] = $element->alt; + $item['content'] = $item['title'] + . '
    ' + . $element + . ''; + + $this->items[] = $item; + + $num++; + if ($num >= $max) + break 2; + } + } + } + + public function getURI(){ + if(!is_null($this->getInput('s')) && !is_null($this->getInput('r')) && !is_null($this->getInput('c'))) { + $subcategory = $this->getInput('s'); + $link = self::URI + . $this->getInput('r') + . '/' + . $this->getInput('c') + . '/' + . $subcategory; + + return $link; + } + + return parent::getURI(); + } + + public function getName(){ + if(!is_null($this->getInput('s'))) { + $subcategory = $this->getInput('s'); + return 'PickyWallpapers - ' + . $this->getInput('c') + . ($subcategory ? ' > ' . $subcategory : '') + . ' [' + . $this->getInput('r') + . ']'; + } + + return parent::getName(); + } +} diff --git a/bridges/PinterestBridge.php b/bridges/PinterestBridge.php new file mode 100644 index 0000000..c5282ff --- /dev/null +++ b/bridges/PinterestBridge.php @@ -0,0 +1,163 @@ + array( + 'u' => array( + 'name' => 'username', + 'required' => true + ), + 'b' => array( + 'name' => 'board', + 'required' => true + ), + 'r' => array( + 'name' => 'Use custom RSS', + 'type' => 'checkbox', + 'required' => false, + 'title' => 'Uncheck to return data via custom filters (more data)' + ) + ), + 'From search' => array( + 'q' => array( + 'name' => 'Keyword', + 'required' => true + ) + ) + ); + + public function collectData(){ + switch($this->queriedContext) { + case 'By username and board': + if($this->getInput('r')) { + $html = getSimpleHTMLDOMCached($this->getURI()); + $this->getUserResults($html); + } else { + $this->collectExpandableDatas($this->getURI() . '.rss'); + } + break; + case 'From search': + default: + $html = getSimpleHTMLDOMCached($this->getURI()); + $this->getSearchResults($html); + } + } + + private function getUserResults($html){ + $json = json_decode($html->find('#jsInit1', 0)->innertext, true); + $results = $json['tree']['children'][0]['children'][0]['children'][0]['options']['props']['data']['board_feed']; + $username = $json['resourceDataCache'][0]['data']['owner']['username']; + $fullname = $json['resourceDataCache'][0]['data']['owner']['full_name']; + $avatar = $json['resourceDataCache'][0]['data']['owner']['image_small_url']; + + foreach($results as $result) { + $item = array(); + + $item['uri'] = $result['link']; + + // Some use regular titles, others provide 'advanced' infos, a few + // provide even less info. Thus we attempt multiple options. + $item['title'] = trim($result['title']); + + if($item['title'] === "") + $item['title'] = trim($result['rich_summary']['display_name']); + + if($item['title'] === "") + $item['title'] = trim($result['description']); + + $item['timestamp'] = strtotime($result['created_at']); + $item['username'] = $username; + $item['fullname'] = $fullname; + $item['avatar'] = $avatar; + $item['author'] = $item['username'] . ' (' . $item['fullname'] . ')'; + $item['content'] = '

    ' + . $item['username'] + . '
    ' + . $item['fullname'] + . '



    ' + . $result['description'] + . '

    '; + + $item['enclosures'] = array($result['images']['orig']['url']); + + $this->items[] = $item; + } + } + + private function getSearchResults($html){ + $json = json_decode($html->find('#jsInit1', 0)->innertext, true); + $results = $json['resourceDataCache'][0]['data']['results']; + + foreach($results as $result) { + $item = array(); + + $item['uri'] = self::URI . $result['board']['url']; + + // Some use regular titles, others provide 'advanced' infos, a few + // provide even less info. Thus we attempt multiple options. + $item['title'] = trim($result['title']); + + if($item['title'] === "") + $item['title'] = trim($result['rich_summary']['display_name']); + + if($item['title'] === "") + $item['title'] = trim($result['grid_description']); + + $item['timestamp'] = strtotime($result['created_at']); + $item['username'] = $result['pinner']['username']; + $item['fullname'] = $result['pinner']['full_name']; + $item['avatar'] = $result['pinner']['image_small_url']; + $item['author'] = $item['username'] . ' (' . $item['fullname'] . ')'; + $item['content'] = '

    ' + . $item['username'] + . '
    ' + . $item['fullname'] + . '



    ' + . $result['description'] + . '

    '; + + $item['enclosures'] = array($result['images']['orig']['url']); + + $this->items[] = $item; + } + } + + public function getURI(){ + switch($this->queriedContext) { + case 'By username and board': + $uri = self::URI . '/' . urlencode($this->getInput('u')) . '/' . urlencode($this->getInput('b'));// . '.rss'; + break; + case 'From search': + $uri = self::URI . '/search/?q=' . urlencode($this->getInput('q')); + break; + default: return parent::getURI(); + } + return $uri; + } + + public function getName(){ + switch($this->queriedContext) { + case 'By username and board': + $specific = $this->getInput('u') . ' - ' . $this->getInput('b'); + break; + case 'From search': + $specific = $this->getInput('q'); + break; + default: return parent::getName(); + } + return $specific . ' - ' . self::NAME; + } +} diff --git a/bridges/PlanetLibreBridge.php b/bridges/PlanetLibreBridge.php new file mode 100644 index 0000000..03a6024 --- /dev/null +++ b/bridges/PlanetLibreBridge.php @@ -0,0 +1,38 @@ +find('div[class="post-text"]', 0)->innertext; + return $text; + } + + public function collectData(){ + $html = getSimpleHTMLDOM(self::URI) + or returnServerError('Could not request PlanetLibre.'); + $limit = 0; + foreach($html->find('div.post') as $element) { + if($limit < 5) { + $item = array(); + $item['title'] = $element->find('h1', 0)->plaintext; + $item['uri'] = $element->find('a', 0)->href; + $item['timestamp'] = strtotime( + str_replace( + '/', + '-', + $element->find('div[class="post-date"]', 0)->plaintext + ) + ); + + $item['content'] = $this->extractContent($item['uri']); + $this->items[] = $item; + $limit++; + } + } + } +} diff --git a/bridges/RTBFBridge.php b/bridges/RTBFBridge.php new file mode 100644 index 0000000..22cdaf4 --- /dev/null +++ b/bridges/RTBFBridge.php @@ -0,0 +1,66 @@ + array( + 'name' => 'series id', + 'exampleValue' => 9500, + 'required' => true + ) + )); + + public function collectData(){ + $html = ''; + $limit = 10; + $count = 0; + + $html = getSimpleHTMLDOM($this->getURI()) + or returnServerError('Could not request RTBF.'); + + foreach($html->find('section[id!=widget-ml-avoiraussi-] .rtbf-media-grid article') as $element) { + if($count >= $limit) { + break; + } + + $item = array(); + $item['id'] = $element->getAttribute('data-id'); + $item['uri'] = self::URI . 'detail?id=' . $item['id']; + $thumbnailUriSrcSet = explode( + ',', + $element->find('figure .www-img-16by9 img', 0)->getAttribute('data-srcset') + ); + + $thumbnailUriLastSrc = end($thumbnailUriSrcSet); + $thumbnailUri = explode(' ', $thumbnailUriLastSrc)[0]; + $item['title'] = trim($element->find('h3', 0)->plaintext) + . ' - ' + . trim($element->find('h4', 0)->plaintext); + + $item['timestamp'] = strtotime($element->find('time', 0)->getAttribute('datetime')); + $item['content'] = ''; + $this->items[] = $item; + $count++; + } + } + + public function getURI(){ + if(!is_null($this->getInput('c'))) { + return self::URI . 'emissions/detail?id=' . $this->getInput('c'); + } + + return parent::getURI() . 'emissions/'; + } + + public function getName(){ + if(!is_null($this->getInput('c'))) { + return $this->getInput('c') .' - RTBF Bridge'; + } + + return parent::getName(); + } +} diff --git a/bridges/RainbowSixSiegeBridge.php b/bridges/RainbowSixSiegeBridge.php new file mode 100644 index 0000000..302bb89 --- /dev/null +++ b/bridges/RainbowSixSiegeBridge.php @@ -0,0 +1,36 @@ +find('h3 a', 0)->href; + $uri = 'https://rainbow6.ubisoft.com' . $uri; + $item['uri'] = $uri; + $item['title'] = $article->find('h3', 0)->plaintext; + $item['content'] = $article->find('img', 0)->outertext . '
    ' . $article->find('strong', 0)->plaintext; + $item['timestamp'] = strtotime($article->find('p.news_date', 0)->plaintext); + + $this->items[] = $item; + } + } +} diff --git a/bridges/ReadComicsBridge.php b/bridges/ReadComicsBridge.php new file mode 100644 index 0000000..33c8ed9 --- /dev/null +++ b/bridges/ReadComicsBridge.php @@ -0,0 +1,44 @@ + array( + 'name' => 'keywords, separated by semicolons', + 'exampleValue' => 'first list;second list;...', + 'required' => true + ), + )); + + public function collectData(){ + + function parseDateTimestamp($element){ + $guessedDate = $element->find('span', 0)->plaintext; + $guessedDate = strptime($guessedDate, '%m/%d/%Y'); + $timestamp = mktime(0, 0, 0, $guessedDate['tm_mon'] + 1, $guessedDate['tm_mday'], date('Y')); + + return $timestamp; + } + + $keywordsList = explode(";", $this->getInput('q')); + foreach($keywordsList as $keywords) { + $html = $this->getSimpleHTMLDOM(self::URI . 'comic/' . rawurlencode($keywords)) + or $this->returnServerError('Could not request readcomics.tv.'); + + foreach($html->find('li') as $element) { + $item = array(); + $item['uri'] = $element->find('a.ch-name', 0)->href; + $item['id'] = $item['uri']; + $item['timestamp'] = parseDateTimestamp($element); + $item['title'] = $element->find('a.ch-name', 0)->plaintext; + if(isset($item['title'])) + $this->items[] = $item; + } + } + } +} diff --git a/bridges/Releases3DSBridge.php b/bridges/Releases3DSBridge.php new file mode 100644 index 0000000..a7e1778 --- /dev/null +++ b/bridges/Releases3DSBridge.php @@ -0,0 +1,136 @@ +', $xml)) as $element) { + if($limit >= 5) { + break; + } + + if(strpos($element, '') === false) { + continue; + } + + $releasename = extractFromDelimiters($element, '', ''); + if(empty($releasename)) { + continue; + } + + $id = extractFromDelimiters($element, '', ''); + $name = extractFromDelimiters($element, '', ''); + $publisher = extractFromDelimiters($element, '', ''); + $region = extractFromDelimiters($element, '', ''); + $group = extractFromDelimiters($element, '', ''); + $imagesize = extractFromDelimiters($element, '', ''); + $serial = extractFromDelimiters($element, '', ''); + $titleid = extractFromDelimiters($element, '', ''); + $imgcrc = extractFromDelimiters($element, '', ''); + $filename = extractFromDelimiters($element, '', ''); + $trimmedsize = extractFromDelimiters($element, '', ''); + $firmware = extractFromDelimiters($element, '', ''); + $type = extractFromDelimiters($element, '', ''); + $card = extractFromDelimiters($element, '', ''); + + //Retrieve cover art and short desc from IGN? + $ignResult = false; + $ignDescription = ''; + $ignLink = ''; + $ignDate = time(); + $ignCoverArt = ''; + + $ignSearchUrl = 'http://www.ign.com/search?q=' . urlencode($name); + if($ignResult = getSimpleHTMLDOM($ignSearchUrl)) { + $ignCoverArt = $ignResult->find('div.search-item-media', 0)->find('img', 0)->src; + $ignDesc = $ignResult->find('div.search-item-description', 0)->plaintext; + $ignLink = $ignResult->find('div.search-item-sub-title', 0)->find('a', 1)->href; + $ignDate = strtotime(trim($ignResult->find('span.publish-date', 0)->plaintext)); + $ignDescription = '
    ' + . $ignDesc + . ' More at IGN
    '; + } + + //Main section : Release description from 3DS database + $releaseDescription = '

    Release Details

    Release ID: ' . $id + . '
    Game Name: ' . $name + . '
    Publisher: ' . $publisher + . '
    Region: ' . $region + . '
    Group: ' . $group + . '
    Image size: ' . (intval($imagesize) / 8) + . 'MB
    Serial: ' . $serial + . '
    Title ID: ' . $titleid + . '
    Image CRC: ' . $imgcrc + . '
    File Name: ' . $filename + . '
    Release Name: ' . $releasename + . '
    Trimmed size: ' . intval(intval($trimmedsize) / 1048576) + . 'MB
    Firmware: ' . $firmware + . '
    Type: ' . typeToString($type) + . '
    Card: ' . cardToString($card) + . '
    '; + + //Build search links section to facilitate release search using search engines + $releaseNameEncoded = urlencode(str_replace(' ', '+', $releasename)); + $searchLinkGoogle = 'https://google.com/?q=' . $releaseNameEncoded; + $searchLinkDuckDuckGo = 'https://duckduckgo.com/?q=' . $releaseNameEncoded; + $searchLinkQwant = 'https://lite.qwant.com/?q=' . $releaseNameEncoded . '&t=web'; + $releaseSearchLinks = '

    Search this release

    '; + + //Build and add final item with the above three sections + $item = array(); + $item['title'] = $name; + $item['author'] = $publisher; + $item['timestamp'] = $ignDate; + $item['uri'] = empty($ignLink) ? $searchLinkDuckDuckGo : $ignLink; + $item['content'] = $ignDescription . $releaseDescription . $releaseSearchLinks; + $this->items[] = $item; + $limit++; + } + } +} diff --git a/bridges/ReporterreBridge.php b/bridges/ReporterreBridge.php new file mode 100644 index 0000000..db1104c --- /dev/null +++ b/bridges/ReporterreBridge.php @@ -0,0 +1,47 @@ +find('div[style=text-align:justify]') as $e) { + $text = $e->outertext; + } + + $html2->clear(); + unset($html2); + + // Replace all relative urls with absolute ones + $text = preg_replace( + '/(href|src)(\=[\"\'])(?!http)([^"\']+)/ims', + "$1$2" . self::URI . "$3", + $text + ); + + $text = strip_tags($text, '


    '); + return $text; + } + + public function collectData(){ + $html = getSimpleHTMLDOM(self::URI . 'spip.php?page=backend') + or returnServerError('Could not request Reporterre.'); + $limit = 0; + + foreach($html->find('item') as $element) { + if($limit < 5) { + $item = array(); + $item['title'] = html_entity_decode($element->find('title', 0)->plaintext); + $item['timestamp'] = strtotime($element->find('dc:date', 0)->plaintext); + $item['uri'] = $element->find('guid', 0)->innertext; + $item['content'] = html_entity_decode($this->extractContent($item['uri'])); + $this->items[] = $item; + $limit++; + } + } + } +} diff --git a/bridges/Rue89Bridge.php b/bridges/Rue89Bridge.php new file mode 100644 index 0000000..6599122 --- /dev/null +++ b/bridges/Rue89Bridge.php @@ -0,0 +1,25 @@ +collectExpandableDatas('http://api.rue89.nouvelobs.com/feed'); + } +} diff --git a/bridges/Rule34Bridge.php b/bridges/Rule34Bridge.php new file mode 100644 index 0000000..b46ec00 --- /dev/null +++ b/bridges/Rule34Bridge.php @@ -0,0 +1,12 @@ +find('article') as $article) { + $item = array(); + $item['uri'] = self::URI . $article->find('p.summary a', 0)->href; + $item['title'] = $article->find('header h1 a', 0)->innertext; + + // remove text "En savoir plus" from anecdote content + $article->find('span.read-more', 0)->outertext = ''; + $content = $article->find('p.summary a', 0)->innertext; + + // remove superfluous spaces at the end + $content = substr($content, 0, strlen($content) - 17); + + // get publication date + $str_date = $article->find('time', 0)->datetime; + list($date, $time) = explode(' ', $str_date); + list($y, $m, $d) = explode('-', $date); + list($h, $i) = explode(':', $time); + $timestamp = mktime($h, $i, 0, $m, $d, $y); + $item['timestamp'] = $timestamp; + + $item['content'] = $content; + $this->items[] = $item; + } + } +} diff --git a/bridges/ScoopItBridge.php b/bridges/ScoopItBridge.php new file mode 100644 index 0000000..997837d --- /dev/null +++ b/bridges/ScoopItBridge.php @@ -0,0 +1,42 @@ + array( + 'name' => 'keyword', + 'required' => true + ) + )); + + public function collectData(){ + $this->request = $this->getInput('u'); + $link = self::URI . 'search?q=' . urlencode($this->getInput('u')); + + $html = getSimpleHTMLDOM($link) + or returnServerError('Could not request ScoopIt. for : ' . $link); + + foreach($html->find('div.post-view') as $element) { + $item = array(); + $item['uri'] = $element->find('a', 0)->href; + $item['title'] = preg_replace( + '~[[:cntrl:]]~', + '', + $element->find('div.tCustomization_post_title', 0)->plaintext + ); + + $item['content'] = preg_replace( + '~[[:cntrl:]]~', + '', + $element->find('div.tCustomization_post_description', 0)->plaintext + ); + + $this->items[] = $item; + } + } +} diff --git a/bridges/SensCritiqueBridge.php b/bridges/SensCritiqueBridge.php new file mode 100644 index 0000000..7ac35f2 --- /dev/null +++ b/bridges/SensCritiqueBridge.php @@ -0,0 +1,97 @@ + array( + 'name' => 'Movies', + 'type' => 'checkbox' + ), + 's' => array( + 'name' => 'Series', + 'type' => 'checkbox' + ), + 'g' => array( + 'name' => 'Video Games', + 'type' => 'checkbox' + ), + 'b' => array( + 'name' => 'Books', + 'type' => 'checkbox' + ), + 'bd' => array( + 'name' => 'BD', + 'type' => 'checkbox' + ), + 'mu' => array( + 'name' => 'Music', + 'type' => 'checkbox' + ) + )); + + public function collectData(){ + $categories = array(); + foreach(self::PARAMETERS[$this->queriedContext] as $category => $properties) { + if($this->getInput($category)) { + $uri = self::URI; + switch($category) { + case 'm': $uri .= 'films/cette-semaine'; + break; + case 's': $uri .= 'series/actualite'; + break; + case 'g': $uri .= 'jeuxvideo/actualite'; + break; + case 'b': $uri .= 'livres/actualite'; + break; + case 'bd': $uri .= 'bd/actualite'; + break; + case 'mu': $uri .= 'musique/actualite'; + break; + } + $html = getSimpleHTMLDOM($uri) + or returnServerError('No results for this query.'); + $list = $html->find('ul.elpr-list', 0); + + $this->extractDataFromList($list); + } + } + } + + private function extractDataFromList($list){ + if($list === null) { + returnClientError('Cannot extract data from list'); + } + + foreach($list->find('li') as $movie) { + $item = array(); + $item['author'] = htmlspecialchars_decode($movie->find('.elco-title a', 0)->plaintext, ENT_QUOTES) + . ' ' + . $movie->find('.elco-date', 0)->plaintext; + + $item['title'] = $movie->find('.elco-title a', 0)->plaintext + . ' ' + . $movie->find('.elco-date', 0)->plaintext; + + $item['content'] = '' + . $movie->find('.elco-original-title', 0)->plaintext + . '

    ' + . $movie->find('.elco-baseline', 0)->plaintext + . '
    ' + . $movie->find('.elco-baseline', 1)->plaintext + . '

    ' + . $movie->find('.elco-description', 0)->plaintext + . '

    ' + . trim($movie->find('.erra-ratings .erra-global', 0)->plaintext) + . ' / 10'; + + $item['id'] = $this->getURI() . $movie->find('.elco-title a', 0)->href; + $item['uri'] = $this->getURI() . $movie->find('.elco-title a', 0)->href; + $this->items[] = $item; + } + } +} diff --git a/bridges/SexactuBridge.php b/bridges/SexactuBridge.php new file mode 100644 index 0000000..5bc552a --- /dev/null +++ b/bridges/SexactuBridge.php @@ -0,0 +1,88 @@ + 'href', + 'src' => 'src', + 'data-original' => 'src' + ); + + public function getURI(){ + return self::URI . '/sexactu'; + } + + public function collectData(){ + $html = getSimpleHTMLDOM($this->getURI()) + or returnServerError('Could not request ' . $this->getURI()); + + $sexactu = $html->find('.container_sexactu', 0); + $rowList = $sexactu->find('.row'); + foreach($rowList as $row) { + // only use first list as second one only contains pages numbers + + $title = $row->find('.title', 0); + if($title) { + $item = array(); + $item['author'] = self::AUTHOR; + $item['title'] = $title->plaintext; + $urlAttribute = "data-href"; + $uri = $title->$urlAttribute; + if($uri === false) + continue; + if(substr($uri, 0, 1) === 'h') { // absolute uri + $item['uri'] = $uri; + } else if(substr($uri, 0, 1) === '/') { // domain relative url + $item['uri'] = self::URI . $uri; + } else { + $item['uri'] = $this->getURI() . $uri; + } + $article = $this->loadFullArticle($item['uri']); + $item['content'] = $this->replaceUriInHtmlElement($article->find('.article_content', 0)); + + $publicationDate = $article->find('time[itemprop=datePublished]', 0); + $short_date = $publicationDate->datetime; + $item['timestamp'] = strtotime($short_date); + } else { + // Sometimes we get rubbish, ignore. + continue; + } + $this->items[] = $item; + } + } + + /** + * Loads the full article and returns the contents + * @param $uri The article URI + * @return The article content + */ + private function loadFullArticle($uri){ + $html = getSimpleHTMLDOMCached($uri); + + $content = $html->find('#article', 0); + if($content) { + return $content; + } + + return null; + } + + /** + * Replaces all relative URIs with absolute ones + * @param $element A simplehtmldom element + * @return The $element->innertext with all URIs replaced + */ + private function replaceUriInHtmlElement($element){ + $returned = $element->innertext; + foreach (self::REPLACED_ATTRIBUTES as $initial => $final) { + $returned = str_replace($initial . '="/', $final . '="' . self::URI . '/', $returned); + } + return $returned; + } +} diff --git a/bridges/ShanaprojectBridge.php b/bridges/ShanaprojectBridge.php new file mode 100644 index 0000000..e86f772 --- /dev/null +++ b/bridges/ShanaprojectBridge.php @@ -0,0 +1,123 @@ +getURI() . '/seasons'); + if(!$html) + returnServerError('Could not load \'seasons\' page!'); + + $season = $html->find('div.follows_menu/a', 1); + if(!$season) + returnServerError('Could not find \'Season Anime List\'!'); + + $html = getSimpleHTMLDOM($this->getURI() . $season->href); + if(!$html) + returnServerError( + 'Could not load \'Season Anime List\' from \'' + . $season->innertext + . '\'!' + ); + + return $html; + } + + // Extracts the anime title + private function extractAnimeTitle($anime){ + $title = $anime->find('a', 0); + if(!$title) + returnServerError('Could not find anime title!'); + return trim($title->innertext); + } + + // Extracts the anime URI + private function extractAnimeUri($anime){ + $uri = $anime->find('a', 0); + if(!$uri) + returnServerError('Could not find anime URI!'); + return $this->getURI() . $uri->href; + } + + // Extracts the anime release date (timestamp) + private function extractAnimeTimestamp($anime){ + $timestamp = $anime->find('span.header_info_block', 1); + if(!$timestamp) + return null; + return strtotime($timestamp->innertext); + } + + // Extracts the anime studio name (author) + private function extractAnimeAuthor($anime){ + $author = $anime->find('span.header_info_block', 2); + if(!$author) + return; // Sometimes the studio is unknown, so leave empty + return trim($author->innertext); + } + + // Extracts the episode information (x of y released) + private function extractAnimeEpisodeInformation($anime){ + $episode = $anime->find('div.header_info_episode', 0); + if(!$episode) + returnServerError('Could not find anime episode information!'); + return preg_replace('/\r|\n/', ' ', $episode->plaintext); + } + + // Extracts the background image + private function extractAnimeBackgroundImage($anime){ + // Getting the picture is a little bit tricky as it is part of the style. + // Luckily the style is part of the parent div :) + + if(preg_match("/url\(\/\/([^\)]+)\)/i", $anime->parent->style, $matches)) + return $matches[1]; + + returnServerError('Could not extract background image!'); + } + + // Builds an URI to search for a specific anime (subber is left empty) + private function buildAnimeSearchUri($anime){ + return $this->getURI() + . '/search/?title=' + . urlencode($this->extractAnimeTitle($anime)) + . '&subber='; + } + + // Builds the content string for a given anime + private function buildAnimeContent($anime){ + // We'll use a template string to place our contents + return '    ' + . htmlspecialchars($this->extractAnimeTitle($anime)) + . '

    ' + . $this->extractAnimeEpisodeInformation($anime) + . '


    Search episodes

    '; + } + + public function collectData(){ + $html = $this->loadSeasonAnimeList(); + + $animes = $html->find('div.header_display_box_info'); + if(!$animes) + returnServerError('Could not find anime headers!'); + + foreach($animes as $anime) { + $item = array(); + $item['title'] = $this->extractAnimeTitle($anime); + $item['author'] = $this->extractAnimeAuthor($anime); + $item['uri'] = $this->extractAnimeUri($anime); + $item['timestamp'] = $this->extractAnimeTimestamp($anime); + $item['content'] = $this->buildAnimeContent($anime); + $this->items[] = $item; + } + } +} diff --git a/bridges/Shimmie2Bridge.php b/bridges/Shimmie2Bridge.php new file mode 100644 index 0000000..efbcd9b --- /dev/null +++ b/bridges/Shimmie2Bridge.php @@ -0,0 +1,39 @@ +getURI() + . 'post/list/' + . $this->getInput('t') + . '/' + . $this->getInput('p'); + } + + protected function getItemFromElement($element){ + $item = array(); + $item['uri'] = $this->getURI() . $element->href; + $item['id'] = (int)preg_replace("/[^0-9]/", '', $element->getAttribute(static::IDATTRIBUTE)); + $item['timestamp'] = time(); + $thumbnailUri = $this->getURI() . $element->find('img', 0)->src; + $item['tags'] = $element->getAttribute('data-tags'); + $item['title'] = $this->getName() . ' | ' . $item['id']; + $item['content'] = '
    Tags: ' + . $item['tags']; + + return $item; + } + +} diff --git a/bridges/SoundcloudBridge.php b/bridges/SoundcloudBridge.php new file mode 100644 index 0000000..92d77da --- /dev/null +++ b/bridges/SoundcloudBridge.php @@ -0,0 +1,64 @@ + array( + 'name' => 'username', + 'required' => true + ) + )); + + const CLIENT_ID = '0aca19eae3843844e4053c6d8fdb7875'; + + public function collectData(){ + + $res = json_decode(getContents( + 'https://api.soundcloud.com/resolve?url=http://www.soundcloud.com/' + . urlencode($this->getInput('u')) + . '&client_id=' + . self::CLIENT_ID + )) or returnServerError('No results for this query'); + + $tracks = json_decode(getContents( + 'https://api.soundcloud.com/users/' + . urlencode($res->id) + . '/tracks?client_id=' + . self::CLIENT_ID + )) or returnServerError('No results for this user'); + + for($i = 0; $i < 10; $i++) { + $item = array(); + $item['author'] = $tracks[$i]->user->username . ' - ' . $tracks[$i]->title; + $item['title'] = $tracks[$i]->user->username . ' - ' . $tracks[$i]->title; + $item['content'] = '    '); //strip thead + $html = stristr($html, ''); //remove tbody + $html = str_get_html(stristr($html, '', true)); //remove last tbody and get back as an array + foreach($html->find('tr') as $element) { + $item = array(); + $item['uri'] = $element->find('td', 2)->find('a', 0)->href; + $item['title'] = $element->find('td', 2)->find('a', 0)->plaintext; + $item['team'] = $element->find('td', 1)->innertext; + $item['timestamp'] = strtotime($element->find('td', 0)->plaintext); + $item['content'] = '' + . $this->seriesTitle + . ' - ' + . $item['title'] + . ' by ' + . $item['team'] + . '
    ' + . $fullhtml->find('div.seriesimg', 0)->innertext + . ''; + + $this->items[] = $item; + } + } + + public function getName(){ + if(!empty($this->seriesTitle)) { + return $this->seriesTitle . ' - ' . static::NAME; + } + + return parent::getName(); + } +} diff --git a/bridges/OpenClassroomsBridge.php b/bridges/OpenClassroomsBridge.php new file mode 100644 index 0000000..5f0daca --- /dev/null +++ b/bridges/OpenClassroomsBridge.php @@ -0,0 +1,49 @@ + array( + 'name' => 'Catégorie', + 'type' => 'list', + 'required' => true, + 'values' => array( + 'Arts & Culture' => 'arts', + 'Code' => 'code', + 'Design' => 'design', + 'Entreprise' => 'business', + 'Numérique' => 'digital', + 'Sciences' => 'sciences', + 'Sciences Humaines' => 'humainities', + 'Systèmes d\'information' => 'it', + 'Autres' => 'others' + ) + ) + )); + + public function getURI(){ + if(!is_null($this->getInput('u'))) { + return self::URI . '/courses?categories=' . $this->getInput('u') . '&title=&sort=updatedAt+desc'; + } + + return parent::getURI(); + } + + public function collectData(){ + $html = getSimpleHTMLDOM($this->getURI()) + or returnServerError('Could not request OpenClassrooms.'); + + foreach($html->find('.courseListItem') as $element) { + $item = array(); + $item['uri'] = self::URI . $element->find('a', 0)->href; + $item['title'] = $element->find('h3', 0)->plaintext; + $item['content'] = $element->find('slidingItem__descriptionContent', 0)->plaintext; + $this->items[] = $item; + } + } +} diff --git a/bridges/ParuVenduImmoBridge.php b/bridges/ParuVenduImmoBridge.php new file mode 100644 index 0000000..a2e2b33 --- /dev/null +++ b/bridges/ParuVenduImmoBridge.php @@ -0,0 +1,102 @@ + array( + 'name' => 'Minimal surface m²', + 'type' => 'number' + ), + 'maxprice' => array( + 'name' => 'Max price', + 'type' => 'number' + ), + 'pa' => array( + 'name' => 'Country code', + 'exampleValue' => 'FR' + ), + 'lo' => array( + 'name' => 'department numbers or postal codes, comma-separated' + ) + )); + + public function collectData(){ + $html = getSimpleHTMLDOM($this->getURI()) + or returnServerError('Could not request paruvendu.'); + + foreach($html->find('div.annonce a') as $element) { + + if(!$element->title) { + continue; + } + + $img = ''; + foreach($element->find('span.img img') as $img) { + if($img->original) { + $img = ''; + } + } + + $desc = $element->find('span.desc')[0]->innertext; + $desc = str_replace("voir l'annonce", '', $desc); + + $price = $element->find('span.price')[0]->innertext; + + list($href) = explode('#', $element->href); + + $item = array(); + $item['uri'] = self::URI . $href; + $item['title'] = $element->title; + $item['content'] = $img . $desc . $price; + $this->items[] = $item; + } + } + + public function getURI(){ + $appartment = '&tbApp=1&tbDup=1&tbChb=1&tbLof=1&tbAtl=1&tbPla=1'; + $maison = '&tbMai=1&tbVil=1&tbCha=1&tbPro=1&tbHot=1&tbMou=1&tbFer=1'; + $link = self::URI + . '/immobilier/annonceimmofo/liste/listeAnnonces?tt=1' + . $appartment + . $maison; + + if($this->getInput('minarea')) { + $link .= '&sur0=' . urlencode($this->getInput('minarea')); + } + + if($this->getInput('maxprice')) { + $link .= '&px1=' . urlencode($this->getInput('maxprice')); + } + + if($this->getInput('pa')) { + $link .= '&pa=' . urlencode($this->getInput('pa')); + } + + if($this->getInput('lo')) { + $link .= '&lo=' . urlencode($this->getInput('lo')); + } + return $link; + } + + public function getName(){ + if(!is_null($this->getInput('minarea'))) { + $request = ''; + $minarea = $this->getInput('minarea'); + if(!empty($minarea)) { + $request .= ' ' . $minarea . ' m2'; + } + $location = $this->getInput('lo'); + if(!empty($location)) { + $request .= ' In: ' . $location; + } + return 'Paru Vendu Immobilier' . $request; + } + + return parent::getName(); + } +} diff --git a/bridges/PcGamerBridge.php b/bridges/PcGamerBridge.php new file mode 100644 index 0000000..e0e55ce --- /dev/null +++ b/bridges/PcGamerBridge.php @@ -0,0 +1,23 @@ +getURI(), 300); + $stories = $html->find('div#popularcontent li.most-popular-item'); + foreach ($stories as $element) { + $item['uri'] = $element->find('a', 0)->href; + $articleHtml = getSimpleHTMLDOMCached($item['uri']); + $item['title'] = $element->find('h4 a', 0)->plaintext; + $item['timestamp'] = strtotime($articleHtml->find('meta[name=pub_date]', 0)->content); + $item['content'] = $articleHtml->find('meta[name=description]', 0)->content; + $item['author'] = $articleHtml->find('a[itemprop=author]', 0)->plaintext; + $this->items[] = $item; + } + } +} diff --git a/bridges/PickyWallpapersBridge.php b/bridges/PickyWallpapersBridge.php new file mode 100644 index 0000000..6c26df7 --- /dev/null +++ b/bridges/PickyWallpapersBridge.php @@ -0,0 +1,101 @@ + array( + 'name' => 'category', + 'required' => true + ), + 's' => array( + 'name' => 'subcategory' + ), + 'm' => array( + 'name' => 'Max number of wallpapers', + 'defaultValue' => 12, + 'type' => 'number' + ), + 'r' => array( + 'name' => 'resolution', + 'exampleValue' => '1920x1200, 1680x1050,…', + 'defaultValue' => '1920x1200', + 'pattern' => '[0-9]{3,4}x[0-9]{3,4}' + ) + )); + + public function collectData(){ + $lastpage = 1; + $num = 0; + $max = $this->getInput('m'); + $resolution = $this->getInput('r'); // Wide wallpaper default + + for($page = 1; $page <= $lastpage; $page++) { + $html = getSimpleHTMLDOM($this->getURI() . '/page-' . $page . '/') + or returnServerError('No results for this query.'); + + if($page === 1) { + preg_match('/page-(\d+)\/$/', $html->find('.pages li a', -2)->href, $matches); + $lastpage = min($matches[1], ceil($max / 12)); + } + + foreach($html->find('.items li img') as $element) { + $item = array(); + $item['uri'] = str_replace('www', 'wallpaper', self::URI) + . '/' + . $resolution + . '/' + . basename($element->src); + + $item['timestamp'] = time(); + $item['title'] = $element->alt; + $item['content'] = $item['title'] + . '
    ' + . $element + . ''; + + $this->items[] = $item; + + $num++; + if ($num >= $max) + break 2; + } + } + } + + public function getURI(){ + if(!is_null($this->getInput('s')) && !is_null($this->getInput('r')) && !is_null($this->getInput('c'))) { + $subcategory = $this->getInput('s'); + $link = self::URI + . $this->getInput('r') + . '/' + . $this->getInput('c') + . '/' + . $subcategory; + + return $link; + } + + return parent::getURI(); + } + + public function getName(){ + if(!is_null($this->getInput('s'))) { + $subcategory = $this->getInput('s'); + return 'PickyWallpapers - ' + . $this->getInput('c') + . ($subcategory ? ' > ' . $subcategory : '') + . ' [' + . $this->getInput('r') + . ']'; + } + + return parent::getName(); + } +} diff --git a/bridges/PinterestBridge.php b/bridges/PinterestBridge.php new file mode 100644 index 0000000..d2dd890 --- /dev/null +++ b/bridges/PinterestBridge.php @@ -0,0 +1,121 @@ + array( + 'u' => array( + 'name' => 'username', + 'required' => true + ), + 'b' => array( + 'name' => 'board', + 'required' => true + ) + ), + 'From search' => array( + 'q' => array( + 'name' => 'Keyword', + 'required' => true + ) + ) + ); + + public function collectData(){ + switch($this->queriedContext) { + case 'By username and board': + $this->collectExpandableDatas($this->getURI() . '.rss'); + $this->fixLowRes(); + break; + case 'From search': + default: + $html = getSimpleHTMLDOMCached($this->getURI()); + $this->getSearchResults($html); + } + } + + private function fixLowRes() { + + $newitems = []; + $pattern = '/https\:\/\/i\.pinimg\.com\/[a-zA-Z0-9]*x\//'; + foreach($this->items as $item) { + + $item['content'] = preg_replace($pattern, 'https://i.pinimg.com/originals/', $item['content']); + $newitems[] = $item; + } + $this->items = $newitems; + + } + + private function getSearchResults($html){ + $json = json_decode($html->find('#jsInit1', 0)->innertext, true); + $results = $json['resourceDataCache'][0]['data']['results']; + + foreach($results as $result) { + $item = array(); + + $item['uri'] = self::URI . $result['board']['url']; + + // Some use regular titles, others provide 'advanced' infos, a few + // provide even less info. Thus we attempt multiple options. + $item['title'] = trim($result['title']); + + if($item['title'] === '') + $item['title'] = trim($result['rich_summary']['display_name']); + + if($item['title'] === '') + $item['title'] = trim($result['grid_description']); + + $item['timestamp'] = strtotime($result['created_at']); + $item['username'] = $result['pinner']['username']; + $item['fullname'] = $result['pinner']['full_name']; + $item['avatar'] = $result['pinner']['image_small_url']; + $item['author'] = $item['username'] . ' (' . $item['fullname'] . ')'; + $item['content'] = '

    ' + . $item['username'] + . '
    ' + . $item['fullname'] + . '



    ' + . $result['description'] + . '

    '; + + $item['enclosures'] = array($result['images']['orig']['url']); + + $this->items[] = $item; + } + } + + public function getURI(){ + switch($this->queriedContext) { + case 'By username and board': + $uri = self::URI . '/' . urlencode($this->getInput('u')) . '/' . urlencode($this->getInput('b'));// . '.rss'; + break; + case 'From search': + $uri = self::URI . '/search/?q=' . urlencode($this->getInput('q')); + break; + default: return parent::getURI(); + } + return $uri; + } + + public function getName(){ + switch($this->queriedContext) { + case 'By username and board': + $specific = $this->getInput('u') . ' - ' . $this->getInput('b'); + break; + case 'From search': + $specific = $this->getInput('q'); + break; + default: return parent::getName(); + } + return $specific . ' - ' . self::NAME; + } +} diff --git a/bridges/PixivBridge.php b/bridges/PixivBridge.php new file mode 100644 index 0000000..3a4cc93 --- /dev/null +++ b/bridges/PixivBridge.php @@ -0,0 +1,73 @@ + array( + 'name' => 'Tag to search', + 'exampleValue' => 'example', + 'required' => true + ), + )); + + + public function collectData(){ + + $html = getContents(static::URI.'search.php?word=' . urlencode($this->getInput('tag'))) + or returnClientError('Unable to query pixiv.net'); + $regex = '/getTimestamp(); + + $item['content'] = ""; + $this->items[] = $item; + } + } + + public function cacheImage($url, $illustId) { + + $url = str_replace('_master1200', '', $url); + $url = str_replace('c/240x240/img-master/', 'img-original/', $url); + $path = CACHE_DIR . '/pixiv_img'; + + if(!is_dir($path)) + mkdir($path, 0755, true); + + if(!is_file($path . '/' . $illustId . '.jpeg')) { + $headers = array('Referer: https://www.pixiv.net/member_illust.php?mode=medium&illust_id=' . $illustId); + $illust = getContents($url, $headers); + if(strpos($illust, '404 Not Found') !== false) { + $illust = getContents(str_replace('jpg', 'png', $url), $headers); + } + file_put_contents($path . '/' . $illustId . '.jpeg', $illust); + } + + return 'cache/pixiv_img/' . $illustId . '.jpeg'; + + } + +} diff --git a/bridges/RTBFBridge.php b/bridges/RTBFBridge.php new file mode 100644 index 0000000..22cdaf4 --- /dev/null +++ b/bridges/RTBFBridge.php @@ -0,0 +1,66 @@ + array( + 'name' => 'series id', + 'exampleValue' => 9500, + 'required' => true + ) + )); + + public function collectData(){ + $html = ''; + $limit = 10; + $count = 0; + + $html = getSimpleHTMLDOM($this->getURI()) + or returnServerError('Could not request RTBF.'); + + foreach($html->find('section[id!=widget-ml-avoiraussi-] .rtbf-media-grid article') as $element) { + if($count >= $limit) { + break; + } + + $item = array(); + $item['id'] = $element->getAttribute('data-id'); + $item['uri'] = self::URI . 'detail?id=' . $item['id']; + $thumbnailUriSrcSet = explode( + ',', + $element->find('figure .www-img-16by9 img', 0)->getAttribute('data-srcset') + ); + + $thumbnailUriLastSrc = end($thumbnailUriSrcSet); + $thumbnailUri = explode(' ', $thumbnailUriLastSrc)[0]; + $item['title'] = trim($element->find('h3', 0)->plaintext) + . ' - ' + . trim($element->find('h4', 0)->plaintext); + + $item['timestamp'] = strtotime($element->find('time', 0)->getAttribute('datetime')); + $item['content'] = ''; + $this->items[] = $item; + $count++; + } + } + + public function getURI(){ + if(!is_null($this->getInput('c'))) { + return self::URI . 'emissions/detail?id=' . $this->getInput('c'); + } + + return parent::getURI() . 'emissions/'; + } + + public function getName(){ + if(!is_null($this->getInput('c'))) { + return $this->getInput('c') .' - RTBF Bridge'; + } + + return parent::getName(); + } +} diff --git a/bridges/RadioMelodieBridge.php b/bridges/RadioMelodieBridge.php new file mode 100644 index 0000000..9b3772b --- /dev/null +++ b/bridges/RadioMelodieBridge.php @@ -0,0 +1,30 @@ +find('div[class=actuitem]'); + foreach($list as $element) { + $item = array(); + + // Get picture URL + $pictureHTML = $element->find('div[class=picture]'); + preg_match( + '/background-image:url\((.*)\);/', + $pictureHTML[0]->getAttribute('style'), + $pictures); + $pictureURL = $pictures[1]; + + $item['enclosures'] = array($pictureURL); + $item['uri'] = self::URI . $element->parent()->href; + $item['title'] = $element->find('h3', 0)->plaintext; + $item['content'] = $element->find('p', 0)->plaintext . '
    '; + $this->items[] = $item; + } + } +} diff --git a/bridges/RainbowSixSiegeBridge.php b/bridges/RainbowSixSiegeBridge.php new file mode 100644 index 0000000..d362bbd --- /dev/null +++ b/bridges/RainbowSixSiegeBridge.php @@ -0,0 +1,36 @@ +find('h3 a', 0)->href; + $uri = 'https://rainbow6.ubisoft.com' . $uri; + $item['uri'] = $uri; + $item['title'] = $article->find('h3', 0)->plaintext; + $item['content'] = $article->find('img', 0)->outertext . '
    ' . $article->find('strong', 0)->plaintext; + $item['timestamp'] = strtotime($article->find('p.news_date', 0)->plaintext); + + $this->items[] = $item; + } + } +} diff --git a/bridges/ReadComicsBridge.php b/bridges/ReadComicsBridge.php new file mode 100644 index 0000000..739e6cc --- /dev/null +++ b/bridges/ReadComicsBridge.php @@ -0,0 +1,44 @@ + array( + 'name' => 'keywords, separated by semicolons', + 'exampleValue' => 'first list;second list;...', + 'required' => true + ), + )); + + public function collectData(){ + + function parseDateTimestamp($element){ + $guessedDate = $element->find('span', 0)->plaintext; + $guessedDate = strptime($guessedDate, '%m/%d/%Y'); + $timestamp = mktime(0, 0, 0, $guessedDate['tm_mon'] + 1, $guessedDate['tm_mday'], date('Y')); + + return $timestamp; + } + + $keywordsList = explode(';', $this->getInput('q')); + foreach($keywordsList as $keywords) { + $html = $this->getSimpleHTMLDOM(self::URI . 'comic/' . rawurlencode($keywords)) + or $this->returnServerError('Could not request readcomics.tv.'); + + foreach($html->find('li') as $element) { + $item = array(); + $item['uri'] = $element->find('a.ch-name', 0)->href; + $item['id'] = $item['uri']; + $item['timestamp'] = parseDateTimestamp($element); + $item['title'] = $element->find('a.ch-name', 0)->plaintext; + if(isset($item['title'])) + $this->items[] = $item; + } + } + } +} diff --git a/bridges/Releases3DSBridge.php b/bridges/Releases3DSBridge.php new file mode 100644 index 0000000..a7e1778 --- /dev/null +++ b/bridges/Releases3DSBridge.php @@ -0,0 +1,136 @@ +', $xml)) as $element) { + if($limit >= 5) { + break; + } + + if(strpos($element, '') === false) { + continue; + } + + $releasename = extractFromDelimiters($element, '', ''); + if(empty($releasename)) { + continue; + } + + $id = extractFromDelimiters($element, '', ''); + $name = extractFromDelimiters($element, '', ''); + $publisher = extractFromDelimiters($element, '', ''); + $region = extractFromDelimiters($element, '', ''); + $group = extractFromDelimiters($element, '', ''); + $imagesize = extractFromDelimiters($element, '', ''); + $serial = extractFromDelimiters($element, '', ''); + $titleid = extractFromDelimiters($element, '', ''); + $imgcrc = extractFromDelimiters($element, '', ''); + $filename = extractFromDelimiters($element, '', ''); + $trimmedsize = extractFromDelimiters($element, '', ''); + $firmware = extractFromDelimiters($element, '', ''); + $type = extractFromDelimiters($element, '', ''); + $card = extractFromDelimiters($element, '', ''); + + //Retrieve cover art and short desc from IGN? + $ignResult = false; + $ignDescription = ''; + $ignLink = ''; + $ignDate = time(); + $ignCoverArt = ''; + + $ignSearchUrl = 'http://www.ign.com/search?q=' . urlencode($name); + if($ignResult = getSimpleHTMLDOM($ignSearchUrl)) { + $ignCoverArt = $ignResult->find('div.search-item-media', 0)->find('img', 0)->src; + $ignDesc = $ignResult->find('div.search-item-description', 0)->plaintext; + $ignLink = $ignResult->find('div.search-item-sub-title', 0)->find('a', 1)->href; + $ignDate = strtotime(trim($ignResult->find('span.publish-date', 0)->plaintext)); + $ignDescription = '
    ' + . $ignDesc + . ' More at IGN
    '; + } + + //Main section : Release description from 3DS database + $releaseDescription = '

    Release Details

    Release ID: ' . $id + . '
    Game Name: ' . $name + . '
    Publisher: ' . $publisher + . '
    Region: ' . $region + . '
    Group: ' . $group + . '
    Image size: ' . (intval($imagesize) / 8) + . 'MB
    Serial: ' . $serial + . '
    Title ID: ' . $titleid + . '
    Image CRC: ' . $imgcrc + . '
    File Name: ' . $filename + . '
    Release Name: ' . $releasename + . '
    Trimmed size: ' . intval(intval($trimmedsize) / 1048576) + . 'MB
    Firmware: ' . $firmware + . '
    Type: ' . typeToString($type) + . '
    Card: ' . cardToString($card) + . '
    '; + + //Build search links section to facilitate release search using search engines + $releaseNameEncoded = urlencode(str_replace(' ', '+', $releasename)); + $searchLinkGoogle = 'https://google.com/?q=' . $releaseNameEncoded; + $searchLinkDuckDuckGo = 'https://duckduckgo.com/?q=' . $releaseNameEncoded; + $searchLinkQwant = 'https://lite.qwant.com/?q=' . $releaseNameEncoded . '&t=web'; + $releaseSearchLinks = '

    Search this release

    '; + + //Build and add final item with the above three sections + $item = array(); + $item['title'] = $name; + $item['author'] = $publisher; + $item['timestamp'] = $ignDate; + $item['uri'] = empty($ignLink) ? $searchLinkDuckDuckGo : $ignLink; + $item['content'] = $ignDescription . $releaseDescription . $releaseSearchLinks; + $this->items[] = $item; + $limit++; + } + } +} diff --git a/bridges/ReporterreBridge.php b/bridges/ReporterreBridge.php new file mode 100644 index 0000000..438c55b --- /dev/null +++ b/bridges/ReporterreBridge.php @@ -0,0 +1,47 @@ +find('div[style=text-align:justify]') as $e) { + $text = $e->outertext; + } + + $html2->clear(); + unset($html2); + + // Replace all relative urls with absolute ones + $text = preg_replace( + '/(href|src)(\=[\"\'])(?!http)([^"\']+)/ims', + '$1$2' . self::URI . '$3', + $text + ); + + $text = strip_tags($text, '


    '); + return $text; + } + + public function collectData(){ + $html = getSimpleHTMLDOM(self::URI . 'spip.php?page=backend') + or returnServerError('Could not request Reporterre.'); + $limit = 0; + + foreach($html->find('item') as $element) { + if($limit < 5) { + $item = array(); + $item['title'] = html_entity_decode($element->find('title', 0)->plaintext); + $item['timestamp'] = strtotime($element->find('dc:date', 0)->plaintext); + $item['uri'] = $element->find('guid', 0)->innertext; + $item['content'] = html_entity_decode($this->extractContent($item['uri'])); + $this->items[] = $item; + $limit++; + } + } + } +} diff --git a/bridges/Rue89Bridge.php b/bridges/Rue89Bridge.php new file mode 100644 index 0000000..72f01eb --- /dev/null +++ b/bridges/Rue89Bridge.php @@ -0,0 +1,25 @@ +collectExpandableDatas('http://api.rue89.nouvelobs.com/feed'); + } +} diff --git a/bridges/Rule34Bridge.php b/bridges/Rule34Bridge.php new file mode 100644 index 0000000..b46ec00 --- /dev/null +++ b/bridges/Rule34Bridge.php @@ -0,0 +1,12 @@ +find('article') as $article) { + $item = array(); + $item['uri'] = self::URI . $article->find('p.summary a', 0)->href; + $item['title'] = $article->find('header h1 a', 0)->innertext; + + // remove text "En savoir plus" from anecdote content + $article->find('span.read-more', 0)->outertext = ''; + $content = $article->find('p.summary a', 0)->innertext; + + // remove superfluous spaces at the end + $content = substr($content, 0, strlen($content) - 17); + + // get publication date + $str_date = $article->find('time', 0)->datetime; + list($date, $time) = explode(' ', $str_date); + list($y, $m, $d) = explode('-', $date); + list($h, $i) = explode(':', $time); + $timestamp = mktime($h, $i, 0, $m, $d, $y); + $item['timestamp'] = $timestamp; + + $item['content'] = $content; + $this->items[] = $item; + } + } +} diff --git a/bridges/ScoopItBridge.php b/bridges/ScoopItBridge.php new file mode 100644 index 0000000..997837d --- /dev/null +++ b/bridges/ScoopItBridge.php @@ -0,0 +1,42 @@ + array( + 'name' => 'keyword', + 'required' => true + ) + )); + + public function collectData(){ + $this->request = $this->getInput('u'); + $link = self::URI . 'search?q=' . urlencode($this->getInput('u')); + + $html = getSimpleHTMLDOM($link) + or returnServerError('Could not request ScoopIt. for : ' . $link); + + foreach($html->find('div.post-view') as $element) { + $item = array(); + $item['uri'] = $element->find('a', 0)->href; + $item['title'] = preg_replace( + '~[[:cntrl:]]~', + '', + $element->find('div.tCustomization_post_title', 0)->plaintext + ); + + $item['content'] = preg_replace( + '~[[:cntrl:]]~', + '', + $element->find('div.tCustomization_post_description', 0)->plaintext + ); + + $this->items[] = $item; + } + } +} diff --git a/bridges/SensCritiqueBridge.php b/bridges/SensCritiqueBridge.php new file mode 100644 index 0000000..7ac35f2 --- /dev/null +++ b/bridges/SensCritiqueBridge.php @@ -0,0 +1,97 @@ + array( + 'name' => 'Movies', + 'type' => 'checkbox' + ), + 's' => array( + 'name' => 'Series', + 'type' => 'checkbox' + ), + 'g' => array( + 'name' => 'Video Games', + 'type' => 'checkbox' + ), + 'b' => array( + 'name' => 'Books', + 'type' => 'checkbox' + ), + 'bd' => array( + 'name' => 'BD', + 'type' => 'checkbox' + ), + 'mu' => array( + 'name' => 'Music', + 'type' => 'checkbox' + ) + )); + + public function collectData(){ + $categories = array(); + foreach(self::PARAMETERS[$this->queriedContext] as $category => $properties) { + if($this->getInput($category)) { + $uri = self::URI; + switch($category) { + case 'm': $uri .= 'films/cette-semaine'; + break; + case 's': $uri .= 'series/actualite'; + break; + case 'g': $uri .= 'jeuxvideo/actualite'; + break; + case 'b': $uri .= 'livres/actualite'; + break; + case 'bd': $uri .= 'bd/actualite'; + break; + case 'mu': $uri .= 'musique/actualite'; + break; + } + $html = getSimpleHTMLDOM($uri) + or returnServerError('No results for this query.'); + $list = $html->find('ul.elpr-list', 0); + + $this->extractDataFromList($list); + } + } + } + + private function extractDataFromList($list){ + if($list === null) { + returnClientError('Cannot extract data from list'); + } + + foreach($list->find('li') as $movie) { + $item = array(); + $item['author'] = htmlspecialchars_decode($movie->find('.elco-title a', 0)->plaintext, ENT_QUOTES) + . ' ' + . $movie->find('.elco-date', 0)->plaintext; + + $item['title'] = $movie->find('.elco-title a', 0)->plaintext + . ' ' + . $movie->find('.elco-date', 0)->plaintext; + + $item['content'] = '' + . $movie->find('.elco-original-title', 0)->plaintext + . '

    ' + . $movie->find('.elco-baseline', 0)->plaintext + . '
    ' + . $movie->find('.elco-baseline', 1)->plaintext + . '

    ' + . $movie->find('.elco-description', 0)->plaintext + . '

    ' + . trim($movie->find('.erra-ratings .erra-global', 0)->plaintext) + . ' / 10'; + + $item['id'] = $this->getURI() . $movie->find('.elco-title a', 0)->href; + $item['uri'] = $this->getURI() . $movie->find('.elco-title a', 0)->href; + $this->items[] = $item; + } + } +} diff --git a/bridges/SexactuBridge.php b/bridges/SexactuBridge.php new file mode 100644 index 0000000..b0a7174 --- /dev/null +++ b/bridges/SexactuBridge.php @@ -0,0 +1,88 @@ + 'href', + 'src' => 'src', + 'data-original' => 'src' + ); + + public function getURI(){ + return self::URI . '/sexactu'; + } + + public function collectData(){ + $html = getSimpleHTMLDOM($this->getURI()) + or returnServerError('Could not request ' . $this->getURI()); + + $sexactu = $html->find('.container_sexactu', 0); + $rowList = $sexactu->find('.row'); + foreach($rowList as $row) { + // only use first list as second one only contains pages numbers + + $title = $row->find('.title', 0); + if($title) { + $item = array(); + $item['author'] = self::AUTHOR; + $item['title'] = $title->plaintext; + $urlAttribute = 'data-href'; + $uri = $title->$urlAttribute; + if($uri === false) + continue; + if(substr($uri, 0, 1) === 'h') { // absolute uri + $item['uri'] = $uri; + } else if(substr($uri, 0, 1) === '/') { // domain relative url + $item['uri'] = self::URI . $uri; + } else { + $item['uri'] = $this->getURI() . $uri; + } + $article = $this->loadFullArticle($item['uri']); + $item['content'] = $this->replaceUriInHtmlElement($article->find('.article_content', 0)); + + $publicationDate = $article->find('time[itemprop=datePublished]', 0); + $short_date = $publicationDate->datetime; + $item['timestamp'] = strtotime($short_date); + } else { + // Sometimes we get rubbish, ignore. + continue; + } + $this->items[] = $item; + } + } + + /** + * Loads the full article and returns the contents + * @param $uri The article URI + * @return The article content + */ + private function loadFullArticle($uri){ + $html = getSimpleHTMLDOMCached($uri); + + $content = $html->find('#article', 0); + if($content) { + return $content; + } + + return null; + } + + /** + * Replaces all relative URIs with absolute ones + * @param $element A simplehtmldom element + * @return The $element->innertext with all URIs replaced + */ + private function replaceUriInHtmlElement($element){ + $returned = $element->innertext; + foreach (self::REPLACED_ATTRIBUTES as $initial => $final) { + $returned = str_replace($initial . '="/', $final . '="' . self::URI . '/', $returned); + } + return $returned; + } +} diff --git a/bridges/ShanaprojectBridge.php b/bridges/ShanaprojectBridge.php new file mode 100644 index 0000000..6eadcb1 --- /dev/null +++ b/bridges/ShanaprojectBridge.php @@ -0,0 +1,123 @@ +getURI() . '/seasons'); + if(!$html) + returnServerError('Could not load \'seasons\' page!'); + + $season = $html->find('div.follows_menu/a', 1); + if(!$season) + returnServerError('Could not find \'Season Anime List\'!'); + + $html = getSimpleHTMLDOM($this->getURI() . $season->href); + if(!$html) + returnServerError( + 'Could not load \'Season Anime List\' from \'' + . $season->innertext + . '\'!' + ); + + return $html; + } + + // Extracts the anime title + private function extractAnimeTitle($anime){ + $title = $anime->find('a', 0); + if(!$title) + returnServerError('Could not find anime title!'); + return trim($title->innertext); + } + + // Extracts the anime URI + private function extractAnimeUri($anime){ + $uri = $anime->find('a', 0); + if(!$uri) + returnServerError('Could not find anime URI!'); + return $this->getURI() . $uri->href; + } + + // Extracts the anime release date (timestamp) + private function extractAnimeTimestamp($anime){ + $timestamp = $anime->find('span.header_info_block', 1); + if(!$timestamp) + return null; + return strtotime($timestamp->innertext); + } + + // Extracts the anime studio name (author) + private function extractAnimeAuthor($anime){ + $author = $anime->find('span.header_info_block', 2); + if(!$author) + return; // Sometimes the studio is unknown, so leave empty + return trim($author->innertext); + } + + // Extracts the episode information (x of y released) + private function extractAnimeEpisodeInformation($anime){ + $episode = $anime->find('div.header_info_episode', 0); + if(!$episode) + returnServerError('Could not find anime episode information!'); + return preg_replace('/\r|\n/', ' ', $episode->plaintext); + } + + // Extracts the background image + private function extractAnimeBackgroundImage($anime){ + // Getting the picture is a little bit tricky as it is part of the style. + // Luckily the style is part of the parent div :) + + if(preg_match('/url\(\/\/([^\)]+)\)/i', $anime->parent->style, $matches)) + return $matches[1]; + + returnServerError('Could not extract background image!'); + } + + // Builds an URI to search for a specific anime (subber is left empty) + private function buildAnimeSearchUri($anime){ + return $this->getURI() + . '/search/?title=' + . urlencode($this->extractAnimeTitle($anime)) + . '&subber='; + } + + // Builds the content string for a given anime + private function buildAnimeContent($anime){ + // We'll use a template string to place our contents + return '    ' + . htmlspecialchars($this->extractAnimeTitle($anime)) + . '

    ' + . $this->extractAnimeEpisodeInformation($anime) + . '


    Search episodes

    '; + } + + public function collectData(){ + $html = $this->loadSeasonAnimeList(); + + $animes = $html->find('div.header_display_box_info'); + if(!$animes) + returnServerError('Could not find anime headers!'); + + foreach($animes as $anime) { + $item = array(); + $item['title'] = $this->extractAnimeTitle($anime); + $item['author'] = $this->extractAnimeAuthor($anime); + $item['uri'] = $this->extractAnimeUri($anime); + $item['timestamp'] = $this->extractAnimeTimestamp($anime); + $item['content'] = $this->buildAnimeContent($anime); + $this->items[] = $item; + } + } +} diff --git a/bridges/Shimmie2Bridge.php b/bridges/Shimmie2Bridge.php new file mode 100644 index 0000000..bdbc504 --- /dev/null +++ b/bridges/Shimmie2Bridge.php @@ -0,0 +1,39 @@ +getURI() + . 'post/list/' + . $this->getInput('t') + . '/' + . $this->getInput('p'); + } + + protected function getItemFromElement($element){ + $item = array(); + $item['uri'] = $this->getURI() . $element->href; + $item['id'] = (int)preg_replace('/[^0-9]/', '', $element->getAttribute(static::IDATTRIBUTE)); + $item['timestamp'] = time(); + $thumbnailUri = $this->getURI() . $element->find('img', 0)->src; + $item['tags'] = $element->getAttribute('data-tags'); + $item['title'] = $this->getName() . ' | ' . $item['id']; + $item['content'] = '
    Tags: ' + . $item['tags']; + + return $item; + } + +} diff --git a/bridges/SoundcloudBridge.php b/bridges/SoundcloudBridge.php new file mode 100644 index 0000000..bfd97cb --- /dev/null +++ b/bridges/SoundcloudBridge.php @@ -0,0 +1,64 @@ + array( + 'name' => 'username', + 'required' => true + ) + )); + + const CLIENT_ID = '4jkoEFmZEDaqjwJ9Eih4ATNhcH3vMVfp'; + + public function collectData(){ + + $res = json_decode(getContents( + 'https://api.soundcloud.com/resolve?url=http://www.soundcloud.com/' + . urlencode($this->getInput('u')) + . '&client_id=' + . self::CLIENT_ID + )) or returnServerError('No results for this query'); + + $tracks = json_decode(getContents( + 'https://api.soundcloud.com/users/' + . urlencode($res->id) + . '/tracks?client_id=' + . self::CLIENT_ID + )) or returnServerError('No results for this user'); + + for($i = 0; $i < 10; $i++) { + $item = array(); + $item['author'] = $tracks[$i]->user->username . ' - ' . $tracks[$i]->title; + $item['title'] = $tracks[$i]->user->username . ' - ' . $tracks[$i]->title; + $item['content'] = '

    ' - . $deal->find('a[class*='. $selectorLink .']', 0)->innertext + . $deal->find('a[class*=' . $selectorLink . ']', 0)->innertext . '

    ' . $this->getPrice($deal) . $this->getDiscount($deal) . $this->getShipsFrom($deal) . $this->getShippingCost($deal) . $this->GetSource($deal) - . $deal->find('div[class*='. $selectorDescription .']', 0)->innertext + . $deal->find('div[class*=' . $selectorDescription . ']', 0)->innertext . '    		' - . $deal->find('div[class='. $selectorHot .']', 0)->children(0)->outertext + . $deal->find('div[class*=' . $selectorHot . ']', 0) + ->find('span', 1)->outertext . '
    '; - $dealDateDiv = $deal->find('div[class*='. $selectorDate .']', 0) + $dealDateDiv = $deal->find('div[class*=' . $selectorDate . ']', 0) ->find('span[class=hide--toW3]'); $itemDate = end($dealDateDiv)->plaintext; // In case of a Local deal, there is no date, but we can use @@ -327,7 +1214,7 @@ class PepperBridgeAbstract extends BridgeAbstract { { if ($deal->find( 'span[class*=thread-price]', 0) != null) { - return '
    '.$this->i8n('price') .' : ' + return '
    ' . $this->i8n('price') . ' : ' . $deal->find( 'span[class*=thread-price]', 0 )->plaintext @@ -346,11 +1233,11 @@ class PepperBridgeAbstract extends BridgeAbstract { { if ($deal->find('span[class*=cept-shipping-price]', 0) != null) { if ($deal->find('span[class*=cept-shipping-price]', 0)->children(0) != null) { - return '
    '. $this->i8n('shipping') .' : ' + return '
    ' . $this->i8n('shipping') . ' : ' . $deal->find('span[class*=cept-shipping-price]', 0)->children(0)->innertext . '
    '; } else { - return '
    '. $this->i8n('shipping') .' : ' + return '
    ' . $this->i8n('shipping') . ' : ' . $deal->find('span[class*=cept-shipping-price]', 0)->innertext . '
    '; } @@ -366,7 +1253,7 @@ class PepperBridgeAbstract extends BridgeAbstract { private function GetSource($deal) { if ($deal->find('a[class=text--color-greyShade]', 0) != null) { - return '
    '. $this->i8n('origin') .' : ' + return '
    ' . $this->i8n('origin') . ' : ' . $deal->find('a[class=text--color-greyShade]', 0)->outertext . '
    '; } else { @@ -387,7 +1274,7 @@ class PepperBridgeAbstract extends BridgeAbstract { } else { $discount = ''; } - return '
    '. $this->i8n('discount') .' : ' + return '
    ' . $this->i8n('discount') . ' : ' . $deal->find( 'span[class*=mute--text text--lineThrough]', 0 )->plaintext @@ -428,13 +1315,13 @@ class PepperBridgeAbstract extends BridgeAbstract { 'cept-thread-img' ) ); - if ($deal->find('img[class='. $selectorLazy .']', 0) != null) { + if ($deal->find('img[class=' . $selectorLazy . ']', 0) != null) { return json_decode( html_entity_decode( - $deal->find('img[class='. $selectorLazy .']', 0) + $deal->find('img[class=' . $selectorLazy . ']', 0) ->getAttribute('data-lazy-img')))->{'src'}; } else { - return $deal->find('img[class*='. $selectorPlain .']', 0 )->src; + return $deal->find('img[class*=' . $selectorPlain . ']', 0 )->src; } } @@ -453,9 +1340,9 @@ class PepperBridgeAbstract extends BridgeAbstract { 'text--color-greyShade' ) ); - if ($deal->find('span[class='. $selector .']', 0) != null) { + if ($deal->find('span[class=' . $selector . ']', 0) != null) { return '
    ' - . $deal->find('span[class='. $selector .']', 0)->children(2)->plaintext + . $deal->find('span[class=' . $selector . ']', 0)->children(2)->plaintext . '
    '; } else { return ''; @@ -558,12 +1445,12 @@ class PepperBridgeAbstract extends BridgeAbstract { public function getName(){ switch($this->queriedContext) { case $this->i8n('context-keyword'): - return $this->i8n('bridge-name') . ' - '. $this->i8n('title-keyword') .' : '. $this->getInput('q'); + return $this->i8n('bridge-name') . ' - ' . $this->i8n('title-keyword') . ' : ' . $this->getInput('q'); break; case $this->i8n('context-group'): $values = $this->getParameters()[$this->i8n('context-group')]['group']['values']; $group = array_search($this->getInput('group'), $values); - return $this->i8n('bridge-name') . ' - '. $this->i8n('title-group'). ' : '. $group; + return $this->i8n('bridge-name') . ' - ' . $this->i8n('title-group') . ' : ' . $group; break; default: // Return default value return static::NAME; @@ -577,7 +1464,7 @@ class PepperBridgeAbstract extends BridgeAbstract { * the "$lang" class variable in the local class * @return various the local content needed */ - public function i8n($key) + protected function i8n($key) { if (array_key_exists($key, $this->lang)) { return $this->lang[$key]; diff --git a/bridges/DesoutterBridge.php b/bridges/DesoutterBridge.php new file mode 100644 index 0000000..94059f6 --- /dev/null +++ b/bridges/DesoutterBridge.php @@ -0,0 +1,240 @@ + array( + 'news_lang' => array( + 'name' => 'Language', + 'type' => 'list', + 'required' => true, + 'title' => 'Select your language', + 'defaultValue' => 'Corporate', + 'values' => array( + 'Corporate' + => 'https://www.desouttertools.com/about-desoutter/news-events', + 'Česko' + => 'https://www.desouttertools.cz/o-desoutter/aktuality-udalsoti', + 'Deutschland' + => 'https://www.desoutter.de/ueber-desoutter/news-events', + 'España' + => 'https://www.desouttertools.es/sobre-desoutter/noticias-eventos', + 'México' + => 'https://www.desouttertools.mx/acerca-desoutter/noticias-eventos', + 'France' + => 'https://www.desouttertools.fr/a-propos-de-desoutter/actualites-evenements', + 'Magyarország' + => 'https://www.desouttertools.hu/a-desoutter-vallalatrol/hirek-esemenyek', + 'Italia' + => 'https://www.desouttertools.it/su-desoutter/news-eventi', + '日本' + => 'https://www.desouttertools.jp/desotanituite/niyusu-ibento', + '대한민국' + => 'https://www.desouttertools.co.kr/desoteoe-daehaeseo/nyuseu-mic-ibenteu', + 'Polska' + => 'https://www.desouttertools.pl/o-desoutter/aktualnosci-wydarzenia', + 'Brasil' + => 'https://www.desouttertools.com.br/sobre-desoutter/noti%C2%ADcias-eventos', + 'Portugal' + => 'https://www.desouttertools.pt/sobre-desoutter/notIcias-eventos', + 'România' + => 'https://www.desouttertools.ro/despre-desoutter/noutati-evenimente', + 'Российская Федерация' + => 'https://www.desouttertools.com.ru/o-desoutter/novosti-mieropriiatiia', + 'Slovensko' + => 'https://www.desouttertools.sk/o-spolocnosti-desoutter/novinky-udalosti', + 'Slovenija' + => 'https://www.desouttertools.si/o-druzbi-desoutter/novice-dogodki', + 'Sverige' + => 'https://www.desouttertools.se/om-desoutter/nyheter-evenemang', + 'Türkiye' + => 'https://www.desoutter.com.tr/desoutter-hakkinda/haberler-etkinlikler', + '中国' + => 'https://www.desouttertools.com.cn/guan-yu-ma-tou/xin-wen-he-huo-dong', + ) + ), + ), + self::CATEGORY_INDUSTRY => array( + 'industry_lang' => array( + 'name' => 'Language', + 'type' => 'list', + 'required' => true, + 'title' => 'Select your language', + 'defaultValue' => 'Corporate', + 'values' => array( + 'Corporate' + => 'https://www.desouttertools.com/industry-4-0/news', + 'Česko' + => 'https://www.desouttertools.cz/prumysl-4-0/novinky', + 'Deutschland' + => 'https://www.desoutter.de/industrie-4-0/news', + 'España' + => 'https://www.desouttertools.es/industria-4-0/noticias', + 'México' + => 'https://www.desouttertools.mx/industria-4-0/noticias', + 'France' + => 'https://www.desouttertools.fr/industrie-4-0/actualites', + 'Magyarország' + => 'https://www.desouttertools.hu/industry-4-0/hirek', + 'Italia' + => 'https://www.desouttertools.it/industry-4-0/news', + '日本' + => 'https://www.desouttertools.jp/industry-4-0/news', + '대한민국' + => 'https://www.desouttertools.co.kr/industry-4-0/news', + 'Polska' + => 'https://www.desouttertools.pl/przemysl-4-0/wiadomosci', + 'Brasil' + => 'https://www.desouttertools.com.br/industria-4-0/noticias', + 'Portugal' + => 'https://www.desouttertools.pt/industria-4-0/noticias', + 'România' + => 'https://www.desouttertools.ro/industry-4-0/noutati', + 'Российская Федерация' + => 'https://www.desouttertools.com.ru/industry-4-0/news', + 'Slovensko' + => 'https://www.desouttertools.sk/priemysel-4-0/novinky', + 'Slovenija' + => 'https://www.desouttertools.si/industrija-4-0/novice', + 'Sverige' + => 'https://www.desouttertools.se/industri-4-0/nyheter', + 'Türkiye' + => 'https://www.desoutter.com.tr/endustri-4-0/haberler', + '中国' + => 'https://www.desouttertools.com.cn/industry-4-0/news', + ) + ), + ), + 'global' => array( + 'full' => array( + 'name' => 'Load full articles', + 'type' => 'checkbox', + 'required' => false, + 'title' => 'Enable to load the full article for each item' + ) + ) + ); + + private $title; + + public function getURI() { + switch($this->queriedContext) { + case self::CATEGORY_NEWS: + return $this->getInput('news_lang') ?: parent::getURI(); + case self::CATEGORY_INDUSTRY: + return $this->getInput('industry_lang') ?: parent::getURI(); + } + + return parent::getURI(); + } + + public function getName() { + return isset($this->title) ? $this->title . ' - ' . parent::getName() : parent::getName(); + } + + public function collectData() { + + // Uncomment to generate list of languages automtically (dev mode) + /* + switch($this->queriedContext) { + case self::CATEGORY_NEWS: + $this->extractNewsLanguages(); die; + case self::CATEGORY_INDUSTRY: + $this->extractIndustryLanguages(); die; + } + */ + + $html = getSimpleHTMLDOM($this->getURI()) + or returnServerError('Could not request ' . $this->getURI()); + + $html = defaultLinkTo($html, $this->getURI()); + + $this->title = html_entity_decode($html->find('title', 0)->plaintext, ENT_QUOTES); + + foreach($html->find('article') as $article) { + $item = array(); + + $item['uri'] = $article->find('[itemprop="name"]', 0)->href; + $item['title'] = $article->find('[itemprop="name"]', 0)->title; + + if($this->getInput('full')) { + $item['content'] = $this->getFullNewsArticle($item['uri']); + } else { + $item['content'] = $article->find('[itemprop="description"]', 0)->plaintext; + } + + $this->items[] = $item; + } + + } + + private function getFullNewsArticle($uri) { + $html = getSimpleHTMLDOMCached($uri) + or returnServerError('Unable to load full article!'); + + $html = defaultLinkTo($html, $this->getURI()); + + return $html->find('section.article', 0); + } + + /** + * Generates a HTML page with a PHP formatted array of languages, + * pointing to the corresponding news pages. Implementation is based + * on the 'Corporate' site. + * @return void + */ + private function extractNewsLanguages() { + $html = getSimpleHTMLDOMCached('https://www.desouttertools.com/about-desoutter/news-events') + or returnServerError('Error loading news!'); + + $html = defaultLinkTo($html, static::URI); + + $items = $html->find('ul[class="dropdown-menu"] li'); + + $list = "\t'Corporate'\n\t=> 'https://www.desouttertools.com/about-desoutter/news-events',\n"; + + foreach($items as $item) { + $lang = trim($item->plaintext); + $uri = $item->find('a', 0)->href; + + $list .= "\t'{$lang}'\n\t=> '{$uri}',\n"; + } + + echo $list; + } + + /** + * Generates a HTML page with a PHP formatted array of languages, + * pointing to the corresponding news pages. Implementation is based + * on the 'Corporate' site. + * @return void + */ + private function extractIndustryLanguages() { + $html = getSimpleHTMLDOMCached('https://www.desouttertools.com/industry-4-0/news') + or returnServerError('Error loading news!'); + + $html = defaultLinkTo($html, static::URI); + + $items = $html->find('ul[class="dropdown-menu"] li'); + + $list = "\t'Corporate'\n\t=> 'https://www.desouttertools.com/industry-4-0/news',\n"; + + foreach($items as $item) { + $lang = trim($item->plaintext); + $uri = $item->find('a', 0)->href; + + $list .= "\t'{$lang}'\n\t=> '{$uri}',\n"; + } + + echo $list; + } + +} diff --git a/bridges/DevToBridge.php b/bridges/DevToBridge.php new file mode 100644 index 0000000..2dfc033 --- /dev/null +++ b/bridges/DevToBridge.php @@ -0,0 +1,105 @@ + array( + 'tag' => array( + 'name' => 'Tag', + 'type' => 'text', + 'required' => true, + 'title' => 'Insert your tag', + 'exampleValue' => 'python' + ), + 'full' => array( + 'name' => 'Full article', + 'type' => 'checkbox', + 'required' => false, + 'title' => 'Enable to receive the full article for each item', + 'defaultValue' => false + ) + ) + ); + + public function getURI() { + switch($this->queriedContext) { + case self::CONTEXT_BY_TAG: + if($tag = $this->getInput('tag')) { + return static::URI . '/t/' . urlencode($tag); + } + break; + } + + return parent::getURI(); + } + + public function getIcon() { + return 'https://practicaldev-herokuapp-com.freetls.fastly.net/assets/ +apple-icon-5c6fa9f2bce280428589c6195b7f1924206a53b782b371cfe2d02da932c8c173.png'; + } + + public function collectData() { + + $html = getSimpleHTMLDOMCached($this->getURI()) + or returnServerError('Could not request ' . $this->getURI()); + + $html = defaultLinkTo($html, static::URI); + + $articles = $html->find('div[class="single-article"]') + or returnServerError('Could not find articles!'); + + foreach($articles as $article) { + + if($article->find('[class*="cta"]', 0)) { // Skip ads + continue; + } + + $item = array(); + + $item['uri'] = $article->find('a[id*=article-link]', 0)->href; + $item['title'] = $article->find('h3', 0)->plaintext; + + // i.e. "Charlie Harrington・Sep 21" + $item['timestamp'] = strtotime(explode('・', $article->find('h4 a', 0)->plaintext, 2)[1]); + $item['author'] = explode('・', $article->find('h4 a', 0)->plaintext, 2)[0]; + + // Profile image + $item['enclosures'] = array($article->find('img', 0)->src); + + if($this->getInput('full')) { + $fullArticle = $this->getFullArticle($item['uri']); + $item['content'] = << +

    {$fullArticle}

    +EOD; + } else { + $item['content'] = << +

    {$item['title']}

    +EOD; + } + + $item['categories'] = array_map(function($e){ return $e->plaintext; }, $article->find('div.tags span.tag')); + + $this->items[] = $item; + } + + } + + private function getFullArticle($url) { + $html = getSimpleHTMLDOMCached($url) + or returnServerError('Unable to load article from "' . $url . '"!'); + + $html = defaultLinkTo($html, static::URI); + + return $html->find('[id="article-body"]', 0); + } + +} diff --git a/bridges/DiceBridge.php b/bridges/DiceBridge.php index dc6ea15..11218df 100644 --- a/bridges/DiceBridge.php +++ b/bridges/DiceBridge.php @@ -75,6 +75,10 @@ class DiceBridge extends BridgeAbstract { ), )); + public function getIcon() { + return 'https://assets.dice.com/techpro/img/favicons/favicon.ico'; + } + public function collectData() { $uri = 'https://www.dice.com/jobs/advancedResult.html'; $uri .= '?for_one=' . urlencode($this->getInput('for_one')); diff --git a/bridges/DilbertBridge.php b/bridges/DilbertBridge.php index 959a91a..a84e5e8 100644 --- a/bridges/DilbertBridge.php +++ b/bridges/DilbertBridge.php @@ -9,8 +9,8 @@ class DilbertBridge extends BridgeAbstract { public function collectData(){ - $html = getSimpleHTMLDOM($this->getURI()) - or returnServerError('Could not request Dilbert: ' . $this->getURI()); + $html = getSimpleHTMLDOM(self::URI) + or returnServerError('Could not request Dilbert: ' . self::URI); foreach($html->find('section.comic-item') as $element) { diff --git a/bridges/DiscogsBridge.php b/bridges/DiscogsBridge.php index 9fe4f51..534ac08 100644 --- a/bridges/DiscogsBridge.php +++ b/bridges/DiscogsBridge.php @@ -81,7 +81,7 @@ class DiscogsBridge extends BridgeAbstract { . $this->getInput('username_folder') . '/collection/folders/' . $this->getInput('folderid') - .'/releases?sort=added&sort_order=desc') + . '/releases?sort=added&sort_order=desc') or returnServerError('Unable to query discogs !'); $jsonData = json_decode($data, true)['releases']; } diff --git a/bridges/DribbbleBridge.php b/bridges/DribbbleBridge.php index 07c4c6e..5058da6 100644 --- a/bridges/DribbbleBridge.php +++ b/bridges/DribbbleBridge.php @@ -7,6 +7,11 @@ class DribbbleBridge extends BridgeAbstract { const CACHE_TIMEOUT = 1800; const DESCRIPTION = 'Returns the newest popular shots from Dribbble.'; + public function getIcon() { + return 'https://cdn.dribbble.com/assets/ +favicon-63b2904a073c89b52b19aa08cebc16a154bcf83fee8ecc6439968b1e6db569c7.ico'; + } + public function collectData(){ $html = getSimpleHTMLDOM(self::URI . '/shots') or returnServerError('Error while downloading the website content'); diff --git a/bridges/ETTVBridge.php b/bridges/ETTVBridge.php index ab90bf7..c348ca0 100644 --- a/bridges/ETTVBridge.php +++ b/bridges/ETTVBridge.php @@ -94,17 +94,20 @@ class ETTVBridge extends BridgeAbstract { ) )); + protected $results_link; + public function collectData(){ - // No control on inputs, because all have defaultValue set + // No control on inputs, because all defaultValue are set $query_str = 'torrents-search.php'; - $query_str .= '?search=' . urlencode('+'.str_replace(' ', ' +', $this->getInput('query'))); + $query_str .= '?search=' . urlencode('+' . str_replace(' ', ' +', $this->getInput('query'))); $query_str .= '&cat=' . $this->getInput('cat'); - $query_str .= 'incldead&=' . $this->getInput('status'); + $query_str .= '&incldead=' . $this->getInput('status'); $query_str .= '&lang=' . $this->getInput('lang'); $query_str .= '&sort=id&order=desc'; // Get results page - $html = getSimpleHTMLDOM(self::URI . $query_str) + $this->results_link = self::URI . $query_str; + $html = getSimpleHTMLDOM($this->results_link) or returnServerError('Could not request ' . $this->getName()); // Loop on each entry @@ -125,7 +128,7 @@ class ETTVBridge extends BridgeAbstract { $item = array(); $item['author'] = $details->children(6)->children(1)->plaintext; $item['title'] = $entry->title; - $item['uri'] = $dllinks->children(0)->children(0)->children(0)->href; + $item['uri'] = $link; $item['timestamp'] = strtotime($details->children(7)->children(1)->plaintext); $item['content'] = ''; $item['content'] .= '
    Name: ' . $details->children(0)->children(1)->innertext; @@ -139,4 +142,20 @@ class ETTVBridge extends BridgeAbstract { $this->items[] = $item; } } + + public function getName(){ + if($this->getInput('query')) { + return '[' . self::NAME . '] ' . $this->getInput('query'); + } + + return self::NAME; + } + + public function getURI(){ + if(isset($this->results_link) && !empty($this->results_link)) { + return $this->results_link; + } + + return self::URI; + } } diff --git a/bridges/EliteDangerousGalnetBridge.php b/bridges/EliteDangerousGalnetBridge.php index 86a1bbf..d19b360 100644 --- a/bridges/EliteDangerousGalnetBridge.php +++ b/bridges/EliteDangerousGalnetBridge.php @@ -7,6 +7,11 @@ class EliteDangerousGalnetBridge extends BridgeAbstract { const CACHE_TIMEOUT = 7200; // 2h const DESCRIPTION = 'Returns the latest page of news from Galnet'; + public function getIcon() { + return 'https://community.elitedangerous.com/sites/ +EDSITE_COMM/themes/bootstrap/bootstrap_community/favicon.ico'; + } + public function collectData(){ $html = getSimpleHTMLDOM(self::URI) or returnServerError('Error while downloading the website content'); diff --git a/bridges/ElloBridge.php b/bridges/ElloBridge.php index c0e266e..b51fed7 100644 --- a/bridges/ElloBridge.php +++ b/bridges/ElloBridge.php @@ -45,9 +45,10 @@ class ElloBridge extends BridgeAbstract { $item = array(); $item['author'] = $this->getUsername($post, $postData); $item['timestamp'] = strtotime($post->created_at); - $item['title'] = $this->findText($post->summary); + $item['title'] = strip_tags($this->findText($post->summary)); $item['content'] = $this->getPostContent($post->body); $item['enclosures'] = $this->getEnclosures($post, $postData); + $item['uri'] = self::URI . $item['author'] . '/post/' . $post->token; $content = $post->body; $this->items[] = $item; @@ -57,7 +58,7 @@ class ElloBridge extends BridgeAbstract { } - public function findText($path) { + private function findText($path) { foreach($path as $summaryElement) { @@ -71,7 +72,7 @@ class ElloBridge extends BridgeAbstract { } - public function getPostContent($path) { + private function getPostContent($path) { $content = ''; foreach($path as $summaryElement) { @@ -92,7 +93,7 @@ class ElloBridge extends BridgeAbstract { } - public function getEnclosures($post, $postData) { + private function getEnclosures($post, $postData) { $assets = []; foreach($post->links->assets as $asset) { @@ -108,7 +109,7 @@ class ElloBridge extends BridgeAbstract { } - public function getUsername($post, $postData) { + private function getUsername($post, $postData) { foreach($postData->linked->users as $user) { if($user->id == $post->links->author->id) { @@ -118,9 +119,9 @@ class ElloBridge extends BridgeAbstract { } - public function getAPIKey() { + private function getAPIKey() { $cache = Cache::create('FileCache'); - $cache->setPath(CACHE_DIR); + $cache->setPath(PATH_CACHE); $cache->setParameters(['key']); $key = $cache->loadData(); diff --git a/bridges/ElsevierBridge.php b/bridges/ElsevierBridge.php index f6ba7dd..080fe00 100644 --- a/bridges/ElsevierBridge.php +++ b/bridges/ElsevierBridge.php @@ -57,6 +57,10 @@ class ElsevierBridge extends BridgeAbstract { return ''; } + public function getIcon() { + return 'https://cdn.elsevier.io/verona/includes/favicons/favicon-32x32.png'; + } + public function collectData(){ $uri = self::URI . $this->getInput('j') . '/recent-articles/'; $html = getSimpleHTMLDOM($uri) diff --git a/bridges/EstCeQuonMetEnProdBridge.php b/bridges/EstCeQuonMetEnProdBridge.php index db9d1d5..4439d69 100644 --- a/bridges/EstCeQuonMetEnProdBridge.php +++ b/bridges/EstCeQuonMetEnProdBridge.php @@ -7,19 +7,9 @@ class EstCeQuonMetEnProdBridge extends BridgeAbstract { const CACHE_TIMEOUT = 21600; // 6h const DESCRIPTION = 'Should we put a website in production today? (French)'; - public function collectData(){ - function extractFromDelimiters($string, $start, $end){ - if(strpos($string, $start) !== false) { - $section_retrieved = substr($string, strpos($string, $start) + strlen($start)); - $section_retrieved = substr($section_retrieved, 0, strpos($section_retrieved, $end)); - return $section_retrieved; - } - - return false; - } - - $html = getSimpleHTMLDOM($this->getURI()) - or returnServerError('Could not request EstCeQuonMetEnProd: ' . $this->getURI()); + public function collectData() { + $html = getSimpleHTMLDOM(self::URI) + or returnServerError('Could not request EstCeQuonMetEnProd: ' . self::URI); $item = array(); $item['uri'] = $this->getURI() . '#' . date('Y-m-d'); @@ -28,8 +18,8 @@ class EstCeQuonMetEnProdBridge extends BridgeAbstract { $item['timestamp'] = strtotime('today midnight'); $item['content'] = str_replace( 'src="/', - 'src="' . $this->getURI(), - trim(extractFromDelimiters($html->outertext, '', '

    ')) + 'src="' . self::URI, + trim(extractFromDelimiters($html->outertext, '', '