From 550e60ae0c7e181f945a5dd54bd5e0c85939890a Mon Sep 17 00:00:00 2001 From: Johannes 'josch' Schauer Date: Tue, 24 Sep 2019 22:52:22 +0200 Subject: New upstream version 2019-09-12 --- README.md | 211 +- actions/DetectAction.php | 53 + actions/DisplayAction.php | 235 ++ actions/ListAction.php | 56 + app.json | 8 + bridges/AO3Bridge.php | 121 + bridges/AllocineFRBridge.php | 1 - bridges/AmazonBridge.php | 2 - bridges/AmazonPriceTrackerBridge.php | 1 - bridges/AppleMusicBridge.php | 62 + bridges/ArtStationBridge.php | 93 + bridges/Arte7Bridge.php | 3 +- bridges/AsahiShimbunAJWBridge.php | 72 + bridges/AtmoNouvelleAquitaineBridge.php | 4638 ++++++++++++++++++++++++++++ bridges/AutoJMBridge.php | 195 +- bridges/BAEBridge.php | 4 +- bridges/BadDragonBridge.php | 435 +++ bridges/BakaUpdatesMangaReleasesBridge.php | 103 + bridges/BandcampBridge.php | 76 +- bridges/BinanceBridge.php | 103 + bridges/BingSearchBridge.php | 119 + bridges/BrutBridge.php | 157 + bridges/BundesbankBridge.php | 1 - bridges/CNETFranceBridge.php | 63 + bridges/CachetBridge.php | 134 + bridges/CastorusBridge.php | 2 +- bridges/ComboiosDePortugalBridge.php | 22 + bridges/ContainerLinuxReleasesBridge.php | 1 - bridges/CourrierInternationalBridge.php | 2 +- bridges/CuriousCatBridge.php | 109 + bridges/DailymotionBridge.php | 152 +- bridges/DanbooruBridge.php | 2 +- bridges/DavesTrailerPageBridge.php | 27 + bridges/DealabsBridge.php | 11 +- bridges/DemoBridge.php | 46 - bridges/DemonoidBridge.php | 169 - bridges/DesoutterBridge.php | 9 +- bridges/DollbooruBridge.php | 9 - bridges/EconomistBridge.php | 63 + bridges/EliteDangerousGalnetBridge.php | 3 + bridges/ElloBridge.php | 8 +- bridges/EngadgetBridge.php | 26 + bridges/ExtremeDownloadBridge.php | 1 - bridges/FB2Bridge.php | 14 +- bridges/FDroidBridge.php | 7 +- bridges/FabriceBellardBridge.php | 36 + bridges/FacebookBridge.php | 17 +- bridges/FeedExpanderExampleBridge.php | 62 - bridges/FicbookBridge.php | 164 + bridges/FindACrewBridge.php | 15 +- bridges/FurAffinityBridge.php | 918 ++++++ bridges/GBAtempBridge.php | 1 - bridges/GOGBridge.php | 8 +- bridges/GQMagazineBridge.php | 74 +- bridges/GiteaBridge.php | 27 + bridges/GithubIssueBridge.php | 100 +- bridges/GithubSearchBridge.php | 2 +- bridges/GlassdoorBridge.php | 18 +- bridges/GlowficBridge.php | 88 + bridges/GogsBridge.php | 206 ++ bridges/GooglePlusPostBridge.php | 208 -- bridges/HDWallpapersBridge.php | 10 +- bridges/HaveIBeenPwnedBridge.php | 138 + bridges/HeiseBridge.php | 75 + bridges/HentaiHavenBridge.php | 2 +- bridges/HotUKDealsBridge.php | 4 - bridges/IGNBridge.php | 55 + bridges/IndeedBridge.php | 245 ++ bridges/InstagramBridge.php | 90 +- bridges/InstructablesBridge.php | 494 ++- bridges/InternetArchiveBridge.php | 293 ++ bridges/IvooxBridge.php | 128 + bridges/JustETFBridge.php | 1 - bridges/KununuBridge.php | 41 +- bridges/LaCentraleBridge.php | 477 +++ bridges/LeBonCoinBridge.php | 1 + bridges/LeMondeInformatiqueBridge.php | 5 +- bridges/MangareaderBridge.php | 1 - bridges/MastodonBridge.php | 89 + bridges/MediapartBridge.php | 60 + bridges/MozillaBugTrackerBridge.php | 153 + bridges/MozillaSecurityBridge.php | 3 +- bridges/MydealsBridge.php | 4 - bridges/NYTBridge.php | 26 + bridges/NationalGeographicBridge.php | 194 ++ bridges/NineGagBridge.php | 3 - bridges/NotAlwaysBridge.php | 3 +- bridges/NovelUpdatesBridge.php | 2 +- bridges/OnVaSortirBridge.php | 1 - bridges/OneFortuneADayBridge.php | 12 +- bridges/OpenClassroomsBridge.php | 1 - bridges/PatreonBridge.php | 203 ++ bridges/PikabuBridge.php | 56 +- bridges/PinterestBridge.php | 90 +- bridges/PirateCommunityBridge.php | 88 + bridges/QPlayBridge.php | 132 + bridges/RadioMelodieBridge.php | 91 +- bridges/RoadAndTrackBridge.php | 68 + bridges/Rue89Bridge.php | 5 +- bridges/Rule34pahealBridge.php | 17 + bridges/SIMARBridge.php | 63 + bridges/SakugabooruBridge.php | 11 - bridges/ShanaprojectBridge.php | 161 +- bridges/SkimfeedBridge.php | 2 - bridges/SoundcloudBridge.php | 14 +- bridges/SplCenterBridge.php | 64 + bridges/SteamBridge.php | 91 +- bridges/SteamCommunityBridge.php | 191 ++ bridges/StockFilingsBridge.php | 80 + bridges/StoriesIGBridge.php | 47 + bridges/TelegramBridge.php | 301 ++ bridges/TheGuardianBridge.php | 96 + bridges/ThePirateBayBridge.php | 9 +- bridges/TwitchBridge.php | 202 ++ bridges/TwitterBridge.php | 130 +- bridges/UnsplashBridge.php | 59 +- bridges/VMwareSecurityBridge.php | 31 + bridges/VimeoBridge.php | 175 ++ bridges/VkBridge.php | 9 +- bridges/WikiLeaksBridge.php | 2 - bridges/WikipediaBridge.php | 2 - bridges/WiredBridge.php | 102 + bridges/WordPressPluginUpdateBridge.php | 12 - bridges/WorldOfTanksBridge.php | 20 +- bridges/XenForoBridge.php | 32 +- bridges/YoutubeBridge.php | 109 +- cache/pages/.gitkeep | 0 cache/server/.gitkeep | 0 caches/FileCache.php | 72 +- caches/MemcachedCache.php | 115 + caches/SQLiteCache.php | 121 + composer.json | 12 + composer.lock | 26 + config.default.ini.php | 21 + formats/AtomFormat.php | 97 +- formats/HtmlFormat.php | 30 +- formats/JsonFormat.php | 41 +- formats/MrssFormat.php | 134 +- index.php | 308 +- lib/ActionAbstract.php | 33 + lib/ActionFactory.php | 65 + lib/ActionInterface.php | 34 + lib/Bridge.php | 296 -- lib/BridgeAbstract.php | 12 +- lib/BridgeCard.php | 25 +- lib/BridgeFactory.php | 242 ++ lib/BridgeList.php | 17 +- lib/Cache.php | 140 - lib/CacheFactory.php | 149 + lib/CacheInterface.php | 40 +- lib/Configuration.php | 115 +- lib/Exceptions.php | 26 +- lib/FactoryAbstract.php | 70 + lib/FeedExpander.php | 2 +- lib/FeedItem.php | 44 +- lib/Format.php | 166 - lib/FormatFactory.php | 153 + lib/ParameterValidator.php | 8 +- lib/contents.php | 71 +- lib/html.php | 33 +- lib/rssbridge.php | 48 +- static/HtmlFormat.css | 32 +- static/favicon.png | Bin 0 -> 3007 bytes static/favicon.svg | 122 + static/logo.svg | 162 + static/logo_300px.png | Bin 0 -> 11546 bytes static/logo_600px.png | Bin 0 -> 24072 bytes static/style.css | 84 +- vendor/simplehtmldom/LICENSE | 21 + vendor/simplehtmldom/simple_html_dom.php | 2620 ++++++++-------- whitelist.default.txt | 15 + 171 files changed, 16889 insertions(+), 3882 deletions(-) create mode 100644 actions/DetectAction.php create mode 100644 actions/DisplayAction.php create mode 100644 actions/ListAction.php create mode 100644 app.json create mode 100644 bridges/AO3Bridge.php create mode 100644 bridges/AppleMusicBridge.php create mode 100644 bridges/ArtStationBridge.php create mode 100644 bridges/AsahiShimbunAJWBridge.php create mode 100644 bridges/AtmoNouvelleAquitaineBridge.php create mode 100644 bridges/BadDragonBridge.php create mode 100644 bridges/BakaUpdatesMangaReleasesBridge.php create mode 100644 bridges/BinanceBridge.php create mode 100644 bridges/BingSearchBridge.php create mode 100644 bridges/BrutBridge.php create mode 100644 bridges/CNETFranceBridge.php create mode 100644 bridges/CachetBridge.php create mode 100644 bridges/ComboiosDePortugalBridge.php create mode 100644 bridges/CuriousCatBridge.php create mode 100644 bridges/DavesTrailerPageBridge.php delete mode 100644 bridges/DemoBridge.php delete mode 100644 bridges/DemonoidBridge.php delete mode 100644 bridges/DollbooruBridge.php create mode 100644 bridges/EconomistBridge.php create mode 100644 bridges/EngadgetBridge.php create mode 100644 bridges/FabriceBellardBridge.php delete mode 100644 bridges/FeedExpanderExampleBridge.php create mode 100644 bridges/FicbookBridge.php create mode 100644 bridges/FurAffinityBridge.php create mode 100644 bridges/GiteaBridge.php create mode 100644 bridges/GlowficBridge.php create mode 100644 bridges/GogsBridge.php delete mode 100644 bridges/GooglePlusPostBridge.php create mode 100644 bridges/HaveIBeenPwnedBridge.php create mode 100644 bridges/HeiseBridge.php create mode 100644 bridges/IGNBridge.php create mode 100644 bridges/IndeedBridge.php create mode 100644 bridges/InternetArchiveBridge.php create mode 100644 bridges/IvooxBridge.php create mode 100644 bridges/LaCentraleBridge.php create mode 100644 bridges/MastodonBridge.php create mode 100644 bridges/MediapartBridge.php create mode 100644 bridges/MozillaBugTrackerBridge.php create mode 100644 bridges/NYTBridge.php create mode 100644 bridges/NationalGeographicBridge.php create mode 100644 bridges/PatreonBridge.php create mode 100644 bridges/PirateCommunityBridge.php create mode 100644 bridges/QPlayBridge.php create mode 100644 bridges/RoadAndTrackBridge.php create mode 100644 bridges/SIMARBridge.php delete mode 100644 bridges/SakugabooruBridge.php create mode 100644 bridges/SplCenterBridge.php create mode 100644 bridges/SteamCommunityBridge.php create mode 100644 bridges/StockFilingsBridge.php create mode 100644 bridges/StoriesIGBridge.php create mode 100644 bridges/TelegramBridge.php create mode 100644 bridges/TheGuardianBridge.php create mode 100644 bridges/TwitchBridge.php create mode 100644 bridges/VMwareSecurityBridge.php create mode 100644 bridges/VimeoBridge.php create mode 100644 bridges/WiredBridge.php create mode 100644 cache/pages/.gitkeep create mode 100644 cache/server/.gitkeep create mode 100644 caches/MemcachedCache.php create mode 100644 caches/SQLiteCache.php create mode 100644 composer.json create mode 100644 composer.lock create mode 100644 lib/ActionAbstract.php create mode 100644 lib/ActionFactory.php create mode 100644 lib/ActionInterface.php delete mode 100644 lib/Bridge.php create mode 100644 lib/BridgeFactory.php delete mode 100644 lib/Cache.php create mode 100644 lib/CacheFactory.php create mode 100644 lib/FactoryAbstract.php delete mode 100644 lib/Format.php create mode 100644 lib/FormatFactory.php create mode 100644 static/favicon.png create mode 100644 static/favicon.svg create mode 100644 static/logo.svg create mode 100644 static/logo_300px.png create mode 100644 static/logo_600px.png create mode 100644 vendor/simplehtmldom/LICENSE create mode 100644 whitelist.default.txt diff --git a/README.md b/README.md index 4e2c7b1..66632f9 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ -rss-bridge +![RSS-Bridge](static/logo_600px.png) === -[![LICENSE](https://img.shields.io/badge/license-UNLICENSE-blue.svg)](UNLICENSE) [![GitHub release](https://img.shields.io/github/release/rss-bridge/rss-bridge.svg)](https://github.com/rss-bridge/rss-bridge/releases/latest) [![Debian Release](https://img.shields.io/badge/dynamic/json.svg?label=debian%20release&url=https%3A%2F%2Fsources.debian.org%2Fapi%2Fsrc%2Frss-bridge%2F&query=%24.versions%5B0%5D.version&colorB=blue)](https://tracker.debian.org/pkg/rss-bridge) [![Guix Release](https://img.shields.io/badge/guix%20release-unknown-light--gray.svg)](https://www.gnu.org/software/guix/packages/R/) [![Build Status](https://travis-ci.org/RSS-Bridge/rss-bridge.svg?branch=master)](https://travis-ci.org/RSS-Bridge/rss-bridge) [![Docker Build Status](https://img.shields.io/docker/build/rssbridge/rss-bridge.svg)](https://hub.docker.com/r/rssbridge/rss-bridge/) +[![LICENSE](https://img.shields.io/badge/license-UNLICENSE-blue.svg)](UNLICENSE) [![GitHub release](https://img.shields.io/github/release/rss-bridge/rss-bridge.svg?logo=github)](https://github.com/rss-bridge/rss-bridge/releases/latest) [![Debian Release](https://img.shields.io/badge/dynamic/json.svg?logo=debian&label=debian%20release&url=https%3A%2F%2Fsources.debian.org%2Fapi%2Fsrc%2Frss-bridge%2F&query=%24.versions%5B0%5D.version&colorB=blue)](https://tracker.debian.org/pkg/rss-bridge) [![Guix Release](https://img.shields.io/badge/guix%20release-unknown-blue.svg)](https://www.gnu.org/software/guix/packages/R/) [![Build Status](https://travis-ci.org/RSS-Bridge/rss-bridge.svg?branch=master)](https://travis-ci.org/RSS-Bridge/rss-bridge) [![Docker Build Status](https://img.shields.io/docker/build/rssbridge/rss-bridge.svg?logo=docker)](https://hub.docker.com/r/rssbridge/rss-bridge/) RSS-Bridge is a PHP project capable of generating RSS and Atom feeds for websites which don't have one. It can be used on webservers or as stand alone application in CLI mode. @@ -15,7 +15,6 @@ Supported sites/pages (examples) * `DuckDuckGo`: Most recent results from [DuckDuckGo.com](https://duckduckgo.com/) * `Facebook` : Returns the latest posts on a page or profile on [Facebook](https://facebook.com/) * `FlickrExplore` : [Latest interesting images](http://www.flickr.com/explore) from Flickr -* `GooglePlus` : Most recent posts of user timeline * `GoogleSearch` : Most recent results from Google Search * `Identi.ca` : Identica user timeline (Should be compatible with other Pump.io instances) * `Instagram`: Most recent photos from an Instagram user @@ -66,6 +65,7 @@ RSS-Bridge requires PHP 5.6 or higher with following extensions enabled: - [`simplexml`](https://secure.php.net/manual/en/book.simplexml.php) - [`curl`](https://secure.php.net/manual/en/book.curl.php) - [`json`](https://secure.php.net/manual/en/book.json.php) + - [`sqlite3`](http://php.net/manual/en/book.sqlite3.php) (only when using SQLiteCache) Find more information on our [Wiki](https://github.com/rss-bridge/rss-bridge/wiki) @@ -84,7 +84,7 @@ Deploy Thanks to the community, hosting your own instance of RSS-Bridge is as easy as clicking a button! [![Deploy on Scalingo](https://cdn.scalingo.com/deploy/button.svg)](https://my.scalingo.com/deploy?source=https://github.com/sebsauvage/rss-bridge) -[![Deploy to Docker Cloud](https://files.cloud.docker.com/images/deploy-to-dockercloud.svg)](https://cloud.docker.com/stack/deploy/?repo=https://github.com/rss-bridge/rss-bridge) +[![Deploy to Heroku](https://www.herokucdn.com/deploy/button.svg)](https://heroku.com/deploy) Getting involved === @@ -110,98 +110,117 @@ Use this script to generate the list automatically (using the GitHub API): https://gist.github.com/LogMANOriginal/da00cd1e5f0ca31cef8e193509b17fd8 --> - * [16mhz](https://github.com/16mhz) - * [Ahiles3005](https://github.com/Ahiles3005) - * [Albirew](https://github.com/Albirew) - * [AmauryCarrade](https://github.com/AmauryCarrade) - * [AntoineTurmel](https://github.com/AntoineTurmel) - * [ArthurHoaro](https://github.com/ArthurHoaro) - * [Astalaseven](https://github.com/Astalaseven) - * [Astyan-42](https://github.com/Astyan-42) - * [Daiyousei](https://github.com/Daiyousei) - * [Djuuu](https://github.com/Djuuu) - * [Draeli](https://github.com/Draeli) - * [EtienneM](https://github.com/EtienneM) - * [Frenzie](https://github.com/Frenzie) - * [Ginko-Aloe](https://github.com/Ginko-Aloe) - * [Glandos](https://github.com/Glandos) - * [GregThib](https://github.com/GregThib) - * [Grummfy](https://github.com/Grummfy) - * [JackNUMBER](https://github.com/JackNUMBER) - * [JeremyRand](https://github.com/JeremyRand) - * [Jocker666z](https://github.com/Jocker666z) - * [LogMANOriginal](https://github.com/LogMANOriginal) - * [MonsieurPoutounours](https://github.com/MonsieurPoutounours) - * [Nono-m0le](https://github.com/Nono-m0le) - * [ORelio](https://github.com/ORelio) - * [PaulVayssiere](https://github.com/PaulVayssiere) - * [Piranhaplant](https://github.com/Piranhaplant) - * [Riduidel](https://github.com/Riduidel) - * [Roliga](https://github.com/Roliga) - * [Strubbl](https://github.com/Strubbl) - * [TheRadialActive](https://github.com/TheRadialActive) - * [TwizzyDizzy](https://github.com/TwizzyDizzy) - * [WalterBarrett](https://github.com/WalterBarrett) - * [ZeNairolf](https://github.com/ZeNairolf) - * [adamchainz](https://github.com/adamchainz) - * [aledeg](https://github.com/aledeg) - * [alexAubin](https://github.com/alexAubin) - * [az5he6ch](https://github.com/az5he6ch) - * [b1nj](https://github.com/b1nj) - * [benasse](https://github.com/benasse) - * [captn3m0](https://github.com/captn3m0) - * [chemel](https://github.com/chemel) - * [ckiw](https://github.com/ckiw) - * [cnlpete](https://github.com/cnlpete) - * [corenting](https://github.com/corenting) - * [couraudt](https://github.com/couraudt) - * [da2x](https://github.com/da2x) - * [disk0x](https://github.com/disk0x) - * [eMerzh](https://github.com/eMerzh) - * [em92](https://github.com/em92) - * [fluffy-critter](https://github.com/fluffy-critter) - * [fulmeek](https://github.com/fulmeek) - * [griffaurel](https://github.com/griffaurel) - * [hunhejj](https://github.com/hunhejj) - * [j0k3r](https://github.com/j0k3r) - * [jdigilio](https://github.com/jdigilio) - * [kranack](https://github.com/kranack) - * [kraoc](https://github.com/kraoc) - * [laBecasse](https://github.com/laBecasse) - * [lagaisse](https://github.com/lagaisse) - * [lalannev](https://github.com/lalannev) - * [ldidry](https://github.com/ldidry) - * [lorenzos](https://github.com/lorenzos) - * [m0zes](https://github.com/m0zes) - * [matthewseal](https://github.com/matthewseal) - * [mcbyte-it](https://github.com/mcbyte-it) - * [mdemoss](https://github.com/mdemoss) - * [melangue](https://github.com/melangue) - * [metaMMA](https://github.com/metaMMA) - * [mickael-bertrand](https://github.com/mickael-bertrand) - * [mitsukarenai](https://github.com/mitsukarenai) - * [mr-flibble](https://github.com/mr-flibble) - * [mro](https://github.com/mro) - * [mxmehl](https://github.com/mxmehl) - * [nel50n](https://github.com/nel50n) - * [niawag](https://github.com/niawag) - * [pellaeon](https://github.com/pellaeon) - * [pit-fgfjiudghdf](https://github.com/pit-fgfjiudghdf) - * [pitchoule](https://github.com/pitchoule) - * [pmaziere](https://github.com/pmaziere) - * [prysme01](https://github.com/prysme01) - * [quentinus95](https://github.com/quentinus95) - * [qwertygc](https://github.com/qwertygc) - * [regisenguehard](https://github.com/regisenguehard) - * [rogerdc](https://github.com/rogerdc) - * [sebsauvage](https://github.com/sebsauvage) - * [sublimz](https://github.com/sublimz) - * [sysadminstory](https://github.com/sysadminstory) - * [tameroski](https://github.com/tameroski) - * [teromene](https://github.com/teromene) - * [triatic](https://github.com/triatic) - * [wtuuju](https://github.com/wtuuju) - * [yardenac](https://github.com/yardenac) +* [16mhz](https://github.com/16mhz) +* [adamchainz](https://github.com/adamchainz) +* [Ahiles3005](https://github.com/Ahiles3005) +* [Albirew](https://github.com/Albirew) +* [aledeg](https://github.com/aledeg) +* [alex73](https://github.com/alex73) +* [alexAubin](https://github.com/alexAubin) +* [AmauryCarrade](https://github.com/AmauryCarrade) +* [ArthurHoaro](https://github.com/ArthurHoaro) +* [Astalaseven](https://github.com/Astalaseven) +* [Astyan-42](https://github.com/Astyan-42) +* [az5he6ch](https://github.com/az5he6ch) +* [azdkj532](https://github.com/azdkj532) +* [b1nj](https://github.com/b1nj) +* [benasse](https://github.com/benasse) +* [captn3m0](https://github.com/captn3m0) +* [chemel](https://github.com/chemel) +* [ckiw](https://github.com/ckiw) +* [cnlpete](https://github.com/cnlpete) +* [corenting](https://github.com/corenting) +* [couraudt](https://github.com/couraudt) +* [da2x](https://github.com/da2x) +* [Daiyousei](https://github.com/Daiyousei) +* [dawidsowa](https://github.com/dawidsowa) +* [disk0x](https://github.com/disk0x) +* [DJCrashdummy](https://github.com/DJCrashdummy) +* [Djuuu](https://github.com/Djuuu) +* [DnAp](https://github.com/DnAp) +* [Draeli](https://github.com/Draeli) +* [Dreckiger-Dan](https://github.com/Dreckiger-Dan) +* [em92](https://github.com/em92) +* [eMerzh](https://github.com/eMerzh) +* [EtienneM](https://github.com/EtienneM) +* [floviolleau](https://github.com/floviolleau) +* [fluffy-critter](https://github.com/fluffy-critter) +* [Frenzie](https://github.com/Frenzie) +* [fulmeek](https://github.com/fulmeek) +* [Ginko-Aloe](https://github.com/Ginko-Aloe) +* [Glandos](https://github.com/Glandos) +* [GregThib](https://github.com/GregThib) +* [griffaurel](https://github.com/griffaurel) +* [Grummfy](https://github.com/Grummfy) +* [hunhejj](https://github.com/hunhejj) +* [husim0](https://github.com/husim0) +* [IceWreck](https://github.com/IceWreck) +* [j0k3r](https://github.com/j0k3r) +* [JackNUMBER](https://github.com/JackNUMBER) +* [jdigilio](https://github.com/jdigilio) +* [JeremyRand](https://github.com/JeremyRand) +* [Jocker666z](https://github.com/Jocker666z) +* [johnnygroovy](https://github.com/johnnygroovy) +* [killruana](https://github.com/killruana) +* [klimplant](https://github.com/klimplant) +* [kranack](https://github.com/kranack) +* [kraoc](https://github.com/kraoc) +* [l1n](https://github.com/l1n) +* [laBecasse](https://github.com/laBecasse) +* [lagaisse](https://github.com/lagaisse) +* [lalannev](https://github.com/lalannev) +* [Leomaradan](https://github.com/Leomaradan) +* [ldidry](https://github.com/ldidry) +* [Limero](https://github.com/Limero) +* [LogMANOriginal](https://github.com/LogMANOriginal) +* [lorenzos](https://github.com/lorenzos) +* [m0zes](https://github.com/m0zes) +* [matthewseal](https://github.com/matthewseal) +* [mcbyte-it](https://github.com/mcbyte-it) +* [mdemoss](https://github.com/mdemoss) +* [melangue](https://github.com/melangue) +* [metaMMA](https://github.com/metaMMA) +* [mitsukarenai](https://github.com/mitsukarenai) +* [MonsieurPoutounours](https://github.com/MonsieurPoutounours) +* [mr-flibble](https://github.com/mr-flibble) +* [mro](https://github.com/mro) +* [mxmehl](https://github.com/mxmehl) +* [nel50n](https://github.com/nel50n) +* [niawag](https://github.com/niawag) +* [Nono-m0le](https://github.com/Nono-m0le) +* [ObsidianWitch](https://github.com/ObsidianWitch) +* [ORelio](https://github.com/ORelio) +* [PaulVayssiere](https://github.com/PaulVayssiere) +* [pellaeon](https://github.com/pellaeon) +* [Piranhaplant](https://github.com/Piranhaplant) +* [pit-fgfjiudghdf](https://github.com/pit-fgfjiudghdf) +* [pitchoule](https://github.com/pitchoule) +* [pmaziere](https://github.com/pmaziere) +* [Pofilo](https://github.com/Pofilo) +* [prysme01](https://github.com/prysme01) +* [quentinus95](https://github.com/quentinus95) +* [regisenguehard](https://github.com/regisenguehard) +* [Riduidel](https://github.com/Riduidel) +* [rogerdc](https://github.com/rogerdc) +* [Roliga](https://github.com/Roliga) +* [sebsauvage](https://github.com/sebsauvage) +* [somini](https://github.com/somini) +* [squeek502](https://github.com/squeek502) +* [Strubbl](https://github.com/Strubbl) +* [sublimz](https://github.com/sublimz) +* [sysadminstory](https://github.com/sysadminstory) +* [tameroski](https://github.com/tameroski) +* [teromene](https://github.com/teromene) +* [thefranke](https://github.com/thefranke) +* [ThePadawan](https://github.com/ThePadawan) +* [TheRadialActive](https://github.com/TheRadialActive) +* [triatic](https://github.com/triatic) +* [VerifiedJoseph](https://github.com/VerifiedJoseph) +* [WalterBarrett](https://github.com/WalterBarrett) +* [wtuuju](https://github.com/wtuuju) +* [xurxof](https://github.com/xurxof) +* [yardenac](https://github.com/yardenac) +* [ZeNairolf](https://github.com/ZeNairolf) Licenses === diff --git a/actions/DetectAction.php b/actions/DetectAction.php new file mode 100644 index 0000000..86605de --- /dev/null +++ b/actions/DetectAction.php @@ -0,0 +1,53 @@ +userData['url'] + or returnClientError('You must specify a url!'); + + $format = $this->userData['format'] + or returnClientError('You must specify a format!'); + + $bridgeFac = new \BridgeFactory(); + $bridgeFac->setWorkingDir(PATH_LIB_BRIDGES); + + foreach($bridgeFac->getBridgeNames() as $bridgeName) { + + if(!$bridgeFac->isWhitelisted($bridgeName)) { + continue; + } + + $bridge = $bridgeFac->create($bridgeName); + + if($bridge === false) { + continue; + } + + $bridgeParams = $bridge->detectParameters($targetURL); + + if(is_null($bridgeParams)) { + continue; + } + + $bridgeParams['bridge'] = $bridgeName; + $bridgeParams['format'] = $format; + + header('Location: ?action=display&' . http_build_query($bridgeParams), true, 301); + die(); + + } + + returnClientError('No bridge found for given URL: ' . $targetURL); + } +} diff --git a/actions/DisplayAction.php b/actions/DisplayAction.php new file mode 100644 index 0000000..9b4d363 --- /dev/null +++ b/actions/DisplayAction.php @@ -0,0 +1,235 @@ +userData) ? $this->userData['bridge'] : null; + + $format = $this->userData['format'] + or returnClientError('You must specify a format!'); + + $bridgeFac = new \BridgeFactory(); + $bridgeFac->setWorkingDir(PATH_LIB_BRIDGES); + + // whitelist control + if(!$bridgeFac->isWhitelisted($bridge)) { + throw new \Exception('This bridge is not whitelisted', 401); + die; + } + + // Data retrieval + $bridge = $bridgeFac->create($bridge); + + $noproxy = array_key_exists('_noproxy', $this->userData) + && filter_var($this->userData['_noproxy'], FILTER_VALIDATE_BOOLEAN); + + if(defined('PROXY_URL') && PROXY_BYBRIDGE && $noproxy) { + define('NOPROXY', true); + } + + // Cache timeout + $cache_timeout = -1; + if(array_key_exists('_cache_timeout', $this->userData)) { + + if(!CUSTOM_CACHE_TIMEOUT) { + unset($this->userData['_cache_timeout']); + $uri = parse_url($_SERVER['REQUEST_URI'], PHP_URL_PATH) . '?' . http_build_query($this->userData); + header('Location: ' . $uri, true, 301); + die(); + } + + $cache_timeout = filter_var($this->userData['_cache_timeout'], FILTER_VALIDATE_INT); + + } else { + $cache_timeout = $bridge->getCacheTimeout(); + } + + // Remove parameters that don't concern bridges + $bridge_params = array_diff_key( + $this->userData, + array_fill_keys( + array( + 'action', + 'bridge', + 'format', + '_noproxy', + '_cache_timeout', + '_error_time' + ), '') + ); + + // Remove parameters that don't concern caches + $cache_params = array_diff_key( + $this->userData, + array_fill_keys( + array( + 'action', + 'format', + '_noproxy', + '_cache_timeout', + '_error_time' + ), '') + ); + + // Initialize cache + $cacheFac = new CacheFactory(); + $cacheFac->setWorkingDir(PATH_LIB_CACHES); + $cache = $cacheFac->create(Configuration::getConfig('cache', 'type')); + $cache->setScope(''); + $cache->purgeCache(86400); // 24 hours + $cache->setKey($cache_params); + + $items = array(); + $infos = array(); + $mtime = $cache->getTime(); + + if($mtime !== false + && (time() - $cache_timeout < $mtime) + && !Debug::isEnabled()) { // Load cached data + + // Send "Not Modified" response if client supports it + // Implementation based on https://stackoverflow.com/a/10847262 + if(isset($_SERVER['HTTP_IF_MODIFIED_SINCE'])) { + $stime = strtotime($_SERVER['HTTP_IF_MODIFIED_SINCE']); + + if($mtime <= $stime) { // Cached data is older or same + header('Last-Modified: ' . gmdate('D, d M Y H:i:s ', $mtime) . 'GMT', true, 304); + die(); + } + } + + $cached = $cache->loadData(); + + if(isset($cached['items']) && isset($cached['extraInfos'])) { + foreach($cached['items'] as $item) { + $items[] = new \FeedItem($item); + } + + $infos = $cached['extraInfos']; + } + + } else { // Collect new data + + try { + $bridge->setDatas($bridge_params); + $bridge->collectData(); + + $items = $bridge->getItems(); + + // Transform "legacy" items to FeedItems if necessary. + // Remove this code when support for "legacy" items ends! + if(isset($items[0]) && is_array($items[0])) { + $feedItems = array(); + + foreach($items as $item) { + $feedItems[] = new \FeedItem($item); + } + + $items = $feedItems; + } + + $infos = array( + 'name' => $bridge->getName(), + 'uri' => $bridge->getURI(), + 'icon' => $bridge->getIcon() + ); + } catch(Error $e) { + error_log($e); + + $item = new \FeedItem(); + + // Create "new" error message every 24 hours + $this->userData['_error_time'] = urlencode((int)(time() / 86400)); + + // Error 0 is a special case (i.e. "trying to get property of non-object") + if($e->getCode() === 0) { + $item->setTitle( + 'Bridge encountered an unexpected situation! (' + . $this->userData['_error_time'] + . ')' + ); + } else { + $item->setTitle( + 'Bridge returned error ' + . $e->getCode() + . '! (' + . $this->userData['_error_time'] + . ')' + ); + } + + $item->setURI( + (isset($_SERVER['REQUEST_URI']) ? parse_url($_SERVER['REQUEST_URI'], PHP_URL_PATH) : '') + . '?' + . http_build_query($this->userData) + ); + + $item->setTimestamp(time()); + $item->setContent(buildBridgeException($e, $bridge)); + + $items[] = $item; + } catch(Exception $e) { + error_log($e); + + $item = new \FeedItem(); + + // Create "new" error message every 24 hours + $this->userData['_error_time'] = urlencode((int)(time() / 86400)); + + $item->setURI( + (isset($_SERVER['REQUEST_URI']) ? parse_url($_SERVER['REQUEST_URI'], PHP_URL_PATH) : '') + . '?' + . http_build_query($this->userData) + ); + + $item->setTitle( + 'Bridge returned error ' + . $e->getCode() + . '! (' + . $this->userData['_error_time'] + . ')' + ); + $item->setTimestamp(time()); + $item->setContent(buildBridgeException($e, $bridge)); + + $items[] = $item; + } + + // Store data in cache + $cache->saveData(array( + 'items' => array_map(function($i){ return $i->toArray(); }, $items), + 'extraInfos' => $infos + )); + + } + + // Data transformation + try { + $formatFac = new FormatFactory(); + $formatFac->setWorkingDir(PATH_LIB_FORMATS); + $format = $formatFac->create($format); + $format->setItems($items); + $format->setExtraInfos($infos); + $format->setLastModified($cache->getTime()); + $format->display(); + } catch(Error $e) { + error_log($e); + header('Content-Type: text/html', true, $e->getCode()); + die(buildTransformException($e, $bridge)); + } catch(Exception $e) { + error_log($e); + header('Content-Type: text/html', true, $e->getCode()); + die(buildTransformException($e, $bridge)); + } + } +} diff --git a/actions/ListAction.php b/actions/ListAction.php new file mode 100644 index 0000000..92aef0e --- /dev/null +++ b/actions/ListAction.php @@ -0,0 +1,56 @@ +bridges = array(); + $list->total = 0; + + $bridgeFac = new \BridgeFactory(); + $bridgeFac->setWorkingDir(PATH_LIB_BRIDGES); + + foreach($bridgeFac->getBridgeNames() as $bridgeName) { + + $bridge = $bridgeFac->create($bridgeName); + + if($bridge === false) { // Broken bridge, show as inactive + + $list->bridges[$bridgeName] = array( + 'status' => 'inactive' + ); + + continue; + + } + + $status = $bridgeFac->isWhitelisted($bridgeName) ? 'active' : 'inactive'; + + $list->bridges[$bridgeName] = array( + 'status' => $status, + 'uri' => $bridge->getURI(), + 'name' => $bridge->getName(), + 'icon' => $bridge->getIcon(), + 'parameters' => $bridge->getParameters(), + 'maintainer' => $bridge->getMaintainer(), + 'description' => $bridge->getDescription() + ); + + } + + $list->total = count($list->bridges); + + header('Content-Type: application/json'); + echo json_encode($list, JSON_PRETTY_PRINT); + } +} diff --git a/app.json b/app.json new file mode 100644 index 0000000..f184799 --- /dev/null +++ b/app.json @@ -0,0 +1,8 @@ +{ + "service": "Heroku", + "name": "RSS-Bridge", + "description": "RSS-Bridge is a PHP project capable of generating RSS and Atom feeds for websites which don't have one.", + "repository": "https://github.com/RSS-Bridge/rss-bridge", + "keywords": ["php", "rss-bridge", "rss"] +} + diff --git a/bridges/AO3Bridge.php b/bridges/AO3Bridge.php new file mode 100644 index 0000000..9a3b5c8 --- /dev/null +++ b/bridges/AO3Bridge.php @@ -0,0 +1,121 @@ + array( + 'url' => array( + 'name' => 'url', + 'required' => true, + // Example: F/F tag, complete works only + 'exampleValue' => self::URI + . 'works?work_search[complete]=T&tag_id=F*s*F', + ), + ), + 'Bookmarks' => array( + 'user' => array( + 'name' => 'user', + 'required' => true, + // Example: Nyaaru's bookmarks + 'exampleValue' => 'Nyaaru', + ), + ), + 'Work' => array( + 'id' => array( + 'name' => 'id', + 'required' => true, + // Example: latest chapters from A Better Past by LysSerris + 'exampleValue' => '18181853', + ), + ) + ); + + // Feed for lists of works (e.g. recent works, search results, filtered tags, + // bookmarks, series, collections). + private function collectList($url) { + $html = getSimpleHTMLDOM($url) + or returnServerError('could not request AO3'); + $html = defaultLinkTo($html, self::URI); + + foreach($html->find('.index.group > li') as $element) { + $item = array(); + + $title = $element->find('div h4 a', 0); + if (!isset($title)) continue; // discard deleted works + $item['title'] = $title->plaintext; + $item['content'] = $element; + $item['uri'] = $title->href; + + $strdate = $element->find('div p.datetime', 0)->plaintext; + $item['timestamp'] = strtotime($strdate); + + $chapters = $element->find('dl dd.chapters', 0); + // bookmarked series and external works do not have a chapters count + $chapters = (isset($chapters) ? $chapters->plaintext : 0); + $item['uid'] = $item['uri'] . "/$strdate/$chapters"; + + $this->items[] = $item; + } + } + + // Feed for recent chapters of a specific work. + private function collectWork($id) { + $url = self::URI . "/works/$id/navigate"; + $html = getSimpleHTMLDOM($url) + or returnServerError('could not request AO3'); + $html = defaultLinkTo($html, self::URI); + + $this->title = $html->find('h2 a', 0)->plaintext; + + foreach($html->find('ol.index.group > li') as $element) { + $item = array(); + + $item['title'] = $element->find('a', 0)->plaintext; + $item['content'] = $element; + $item['uri'] = $element->find('a', 0)->href; + + $strdate = $element->find('span.datetime', 0)->plaintext; + $strdate = str_replace('(', '', $strdate); + $strdate = str_replace(')', '', $strdate); + $item['timestamp'] = strtotime($strdate); + + $item['uid'] = $item['uri'] . "/$strdate"; + + $this->items[] = $item; + } + + $this->items = array_reverse($this->items); + } + + public function collectData() { + switch($this->queriedContext) { + case 'Bookmarks': + $user = $this->getInput('user'); + $this->title = $user; + $url = self::URI + . '/users/' . $user + . '/bookmarks?bookmark_search[sort_column]=bookmarkable_date'; + return $this->collectList($url); + case 'List': return $this->collectList( + $this->getInput('url') + ); + case 'Work': return $this->collectWork( + $this->getInput('id') + ); + } + } + + public function getName() { + $name = parent::getName() . " $this->queriedContext"; + if (isset($this->title)) $name .= " - $this->title"; + return $name; + } + + public function getIcon() { + return self::URI . '/favicon.ico'; + } +} diff --git a/bridges/AllocineFRBridge.php b/bridges/AllocineFRBridge.php index 50a41ec..17da903 100644 --- a/bridges/AllocineFRBridge.php +++ b/bridges/AllocineFRBridge.php @@ -10,7 +10,6 @@ class AllocineFRBridge extends BridgeAbstract { 'category' => array( 'name' => 'category', 'type' => 'list', - 'required' => true, 'exampleValue' => 'Faux Raccord', 'title' => 'Select your category', 'values' => array( diff --git a/bridges/AmazonBridge.php b/bridges/AmazonBridge.php index c9d4dc9..bcd83dc 100644 --- a/bridges/AmazonBridge.php +++ b/bridges/AmazonBridge.php @@ -16,7 +16,6 @@ class AmazonBridge extends BridgeAbstract { 'sort' => array( 'name' => 'Sort by', 'type' => 'list', - 'required' => false, 'values' => array( 'Relevance' => 'relevanceblender', 'Price: Low to High' => 'price-asc-rank', @@ -29,7 +28,6 @@ class AmazonBridge extends BridgeAbstract { 'tld' => array( 'name' => 'Country', 'type' => 'list', - 'required' => true, 'values' => array( 'Australia' => 'com.au', 'Brazil' => 'com.br', diff --git a/bridges/AmazonPriceTrackerBridge.php b/bridges/AmazonPriceTrackerBridge.php index e31a03b..6fa11c9 100644 --- a/bridges/AmazonPriceTrackerBridge.php +++ b/bridges/AmazonPriceTrackerBridge.php @@ -19,7 +19,6 @@ class AmazonPriceTrackerBridge extends BridgeAbstract { 'tld' => array( 'name' => 'Country', 'type' => 'list', - 'required' => true, 'values' => array( 'Australia' => 'com.au', 'Brazil' => 'com.br', diff --git a/bridges/AppleMusicBridge.php b/bridges/AppleMusicBridge.php new file mode 100644 index 0000000..5a4f40a --- /dev/null +++ b/bridges/AppleMusicBridge.php @@ -0,0 +1,62 @@ + [ + 'name' => 'Artist URL', + 'exampleValue' => 'https://itunes.apple.com/us/artist/dunderpatrullen/329796274', + 'required' => true, + ], + 'imgSize' => [ + 'name' => 'Image size for thumbnails (in px)', + 'type' => 'number', + 'defaultValue' => 512, + 'required' => true, + ] + ]]; + const CACHE_TIMEOUT = 21600; // 6 hours + + public function collectData() { + $url = $this->getInput('url'); + $html = getSimpleHTMLDOM($url) + or returnServerError('Could not request: ' . $url); + + $imgSize = $this->getInput('imgSize'); + + // Grab the json data from the page + $html = $html->find('script[id=shoebox-ember-data-store]', 0); + $html = strstr($html, '{'); + $html = substr($html, 0, -9); + $json = json_decode($html); + + // Loop through each object + foreach ($json->included as $obj) { + if ($obj->type === 'lockup/album') { + $this->items[] = [ + 'title' => $obj->attributes->artistName . ' - ' . $obj->attributes->name, + 'uri' => $obj->attributes->url, + 'timestamp' => $obj->attributes->releaseDate, + 'enclosures' => $obj->relationships->artwork->data->id, + ]; + } elseif ($obj->type === 'image') { + $images[$obj->id] = $obj->attributes->url; + } + } + + // Add the images to each item + foreach ($this->items as &$item) { + $item['enclosures'] = [ + str_replace('{w}x{h}bb.{f}', $imgSize . 'x0w.jpg', $images[$item['enclosures']]), + ]; + } + + // Sort the order to put the latest albums first + usort($this->items, function($a, $b){ + return $a['timestamp'] < $b['timestamp']; + }); + } +} diff --git a/bridges/ArtStationBridge.php b/bridges/ArtStationBridge.php new file mode 100644 index 0000000..9c12add --- /dev/null +++ b/bridges/ArtStationBridge.php @@ -0,0 +1,93 @@ + array( + 'q' => array( + 'name' => 'Search term', + 'required' => true + ) + ) + ); + + public function getIcon() { + return 'https://www.artstation.com/assets/favicon-58653022bc38c1905ac7aa1b10bffa6b.ico'; + } + + public function getName() { + return self::NAME . ': ' . $this->getInput('q'); + } + + private function fetchSearch($searchQuery) { + $data = '{"query":"' . $searchQuery . '","page":1,"per_page":50,"sorting":"date",'; + $data .= '"pro_first":"1","filters":[],"additional_fields":[]}'; + + $header = array( + 'Content-Type: application/json', + 'Accept: application/json' + ); + + $opts = array( + CURLOPT_POST => true, + CURLOPT_POSTFIELDS => $data, + CURLOPT_RETURNTRANSFER => true + ); + + $jsonSearchURL = self::URI . '/api/v2/search/projects.json'; + $jsonSearchStr = getContents($jsonSearchURL, $header, $opts) + or returnServerError('Could not fetch JSON for search query.'); + return json_decode($jsonSearchStr); + } + + private function fetchProject($hashID) { + $jsonProjectURL = self::URI . '/projects/' . $hashID . '.json'; + $jsonProjectStr = getContents($jsonProjectURL) + or returnServerError('Could not fetch JSON for project.'); + return json_decode($jsonProjectStr); + } + + public function collectData() { + $searchTerm = $this->getInput('q'); + $jsonQuery = $this->fetchSearch($searchTerm); + + foreach($jsonQuery->data as $media) { + // get detailed info about media item + $jsonProject = $this->fetchProject($media->hash_id); + + // create item + $item = array(); + $item['title'] = $media->title; + $item['uri'] = $media->url; + $item['timestamp'] = strtotime($jsonProject->published_at); + $item['author'] = $media->user->full_name; + $item['categories'] = implode(',', $jsonProject->tags); + + $item['content'] = '

' + . $jsonProject->description + . '

'; + + $numAssets = count($jsonProject->assets); + + if ($numAssets > 1) + $item['content'] .= '

Project contains ' + . ($numAssets - 1) + . ' more item(s).

'; + + $this->items[] = $item; + + if (count($this->items) >= 10) + break; + } + } +} diff --git a/bridges/Arte7Bridge.php b/bridges/Arte7Bridge.php index ff72211..562f648 100644 --- a/bridges/Arte7Bridge.php +++ b/bridges/Arte7Bridge.php @@ -91,7 +91,8 @@ class Arte7Bridge extends BridgeAbstract { 'Authorization: Bearer ' . self::API_TOKEN ); - $input = getContents($url, $header) or die('Could not request ARTE.'); + $input = getContents($url, $header) + or returnServerError('Could not request ARTE.'); $input_json = json_decode($input, true); foreach($input_json['videos'] as $element) { diff --git a/bridges/AsahiShimbunAJWBridge.php b/bridges/AsahiShimbunAJWBridge.php new file mode 100644 index 0000000..0ceb038 --- /dev/null +++ b/bridges/AsahiShimbunAJWBridge.php @@ -0,0 +1,72 @@ + array( + 'type' => 'list', + 'name' => 'Section', + 'values' => array( + 'Japan » Social Affairs' => 'japan/social', + 'Japan » People' => 'japan/people', + 'Japan » 3/11 Disaster' => 'japan/0311disaster', + 'Japan » Sci & Tech' => 'japan/sci_tech', + 'Politics' => 'politics', + 'Business' => 'business', + 'Culture » Style' => 'culture/style', + 'Culture » Movies' => 'culture/movies', + 'Culture » Manga & Anime' => 'culture/manga_anime', + 'Asia » China' => 'asia/china', + 'Asia » Korean Peninsula' => 'asia/korean_peninsula', + 'Asia » Around Asia' => 'asia/around_asia', + 'Opinion » Editorial' => 'opinion/editorial', + 'Opinion » Vox Populi' => 'opinion/vox', + ), + 'defaultValue' => 'Politics', + ) + ) + ); + + private function getSectionURI($section) { + return self::getURI() . $section . '/'; + } + + public function collectData() { + $html = getSimpleHTMLDOM($this->getSectionURI($this->getInput('section'))) + or returnServerError('Could not load content'); + + foreach($html->find('#MainInner li a') as $element) { + if ($element->parent()->class == 'HeadlineTopImage-S') { + Debug::log('Skip Headline, it is repeated below'); + continue; + } + $item = array(); + + $item['uri'] = self::BASE_URI . $element->href; + $e_lead = $element->find('span.Lead', 0); + if ($e_lead) { + $item['content'] = $e_lead->innertext; + $e_lead->outertext = ''; + } else { + $item['content'] = $element->innertext; + } + $e_date = $element->find('span.EnDate', 0); + if ($e_date) { + $item['timestamp'] = strtotime($e_date->innertext); + $e_date->outertext = ''; + } + $e_video = $element->find('span.EnVideo', 0); + if ($e_video) { + $e_video->outertext = ''; + $element->innertext = "VIDEO: $element->innertext"; + } + $item['title'] = $element->innertext; + + $this->items[] = $item; + } + } +} diff --git a/bridges/AtmoNouvelleAquitaineBridge.php b/bridges/AtmoNouvelleAquitaineBridge.php new file mode 100644 index 0000000..2ded81a --- /dev/null +++ b/bridges/AtmoNouvelleAquitaineBridge.php @@ -0,0 +1,4638 @@ + array( + 'name' => 'Choisir une ville', + 'type' => 'list', + 'values' => self::CITIES + ) + )); + const CACHE_TIMEOUT = 7200; + + private $dom; + + private function getClosest($search, $arr) { + $closest = null; + foreach ($arr as $key => $value) { + if ($closest === null || abs((int)$search - $closest) > abs((int)$key - (int)$search)) { + $closest = (int)$key; + } + } + return $arr[$closest]; + } + + public function collectData() { + $uri = self::URI . $this->getInput('cities'); + + $html = getSimpleHTMLDOM($uri) + or returnServerError('Could not request ' . $uri); + + $this->dom = $html->find('#block-system-main .city-prevision-map', 0); + + $message = $this->getIndexMessage() . ' ' . $this->getQualityMessage(); + $message .= ' ' . $this->getTomorrowTrendIndexMessage() . ' ' . $this->getTomorrowTrendQualityMessage(); + + $item['uri'] = $uri; + $today = date('d/m/Y'); + $item['title'] = "Bulletin de l'air du $today pour la région Nouvelle Aquitaine."; + $item['title'] .= ' Retrouvez plus d\'informations en allant sur atmo-nouvelleaquitaine.org #QualiteAir.'; + $item['author'] = 'floviolleau'; + $item['content'] = $message; + $item['uid'] = hash('sha256', $item['title']); + + $this->items[] = $item; + } + + private function getIndex() { + $index = $this->dom->find('.indice', 0)->innertext; + + if ($index == 'XX') { + return -1; + } + + return $index; + } + + private function getMaxIndexText() { + // will return '/100' + return $this->dom->find('.pourcent', 0)->innertext; + } + + private function getQualityText($index, $indexes) { + if ($index == -1) { + if (array_key_exists('no-available', $indexes)) { + return $indexes['no-available']; + } + + return 'Aucune donnée'; + } + + return $this->getClosest($index, $indexes); + } + + private function getLegendIndexes() { + $rawIndexes = $this->dom->find('.prevision-legend .prevision-legend-label'); + $indexes = []; + for ($i = 0; $i < count($rawIndexes); $i++) { + if ($rawIndexes[$i]->hasAttribute('data-color')) { + $indexes[$rawIndexes[$i]->getAttribute('data-color')] = $rawIndexes[$i]->innertext; + } + } + + return $indexes; + } + + private function getTomorrowTrendIndex() { + $tomorrowTrendDomNode = $this->dom + ->find('.day-controls.raster-controls .list-raster-controls .raster-control', 2); + $tomorrowTrendIndexNode = null; + + if ($tomorrowTrendDomNode) { + $tomorrowTrendIndexNode = $tomorrowTrendDomNode->find('.raster-control-link', 0); + } + + if ($tomorrowTrendIndexNode && $tomorrowTrendIndexNode->hasAttribute('data-index')) { + $tomorrowTrendIndex = $tomorrowTrendIndexNode->getAttribute('data-index'); + } else { + return -1; + } + + return $tomorrowTrendIndex; + } + + private function getTomorrowTrendQualityText($trendIndex, $indexes) { + if ($trendIndex == -1) { + if (array_key_exists('no-available', $indexes)) { + return $indexes['no-available']; + } + + return 'Aucune donnée'; + } + + return $this->getClosest($trendIndex, $indexes); + } + + private function getIndexMessage() { + $index = $this->getIndex(); + $maxIndexText = $this->getMaxIndexText(); + + if ($index == -1) { + return 'Aucune donnée pour l\'indice.'; + } + + return "L'indice d'aujourd'hui est $index$maxIndexText."; + } + + private function getQualityMessage() { + $index = $index = $this->getIndex(); + $indexes = $this->getLegendIndexes(); + $quality = $this->getQualityText($index, $indexes); + + if ($index == -1) { + return 'Aucune donnée pour la qualité de l\'air.'; + } + + return "La qualité de l'air est $quality."; + } + + private function getTomorrowTrendIndexMessage() { + $trendIndex = $this->getTomorrowTrendIndex(); + $maxIndexText = $this->getMaxIndexText(); + + if ($trendIndex == -1) { + return 'Aucune donnée pour l\'indice prévu demain.'; + } + + return "L'indice prévu pour demain est $trendIndex$maxIndexText."; + } + + private function getTomorrowTrendQualityMessage() { + $trendIndex = $this->getTomorrowTrendIndex(); + $indexes = $this->getLegendIndexes(); + $trendQuality = $this->getTomorrowTrendQualityText($trendIndex, $indexes); + + if ($trendIndex == -1) { + return 'Aucune donnée pour la qualité de l\'air de demain.'; + } + return "La qualite de l'air pour demain sera $trendQuality."; + } + + const CITIES = array( + 'Aast (64460)' => '64001', + 'Abère (64160)' => '64002', + 'Abidos (64150)' => '64003', + 'Abitain (64390)' => '64004', + 'Abjat-sur-Bandiat (24300)' => '24001', + 'Abos (64360)' => '64005', + 'Abzac (16500)' => '16001', + 'Abzac (33230)' => '33001', + 'Accous (64490)' => '64006', + 'Adilly (79200)' => '79002', + 'Adriers (86430)' => '86001', + 'Affieux (19260)' => '19001', + 'Agen (47000)' => '47001', + 'Agmé (47350)' => '47002', + 'Agnac (47800)' => '47003', + 'Agnos (64400)' => '64007', + 'Agonac (24460)' => '24002', + 'Agris (16110)' => '16003', + 'Agudelle (17500)' => '17002', + 'Ahaxe-Alciette-Bascassan (64220)' => '64008', + 'Ahetze (64210)' => '64009', + 'Ahun (23150)' => '23001', + 'Aïcirits-Camou-Suhast (64120)' => '64010', + 'Aiffres (79230)' => '79003', + 'Aignes-et-Puypéroux (16190)' => '16004', + 'Aigonnay (79370)' => '79004', + 'Aigre (16140)' => '16005', + 'Aigrefeuille-d\'Aunis (17290)' => '17003', + 'Aiguillon (47190)' => '47004', + 'Aillas (33124)' => '33002', + 'Aincille (64220)' => '64011', + 'Ainharp (64130)' => '64012', + 'Ainhice-Mongelos (64220)' => '64013', + 'Ainhoa (64250)' => '64014', + 'Aire-sur-l\'Adour (40800)' => '40001', + 'Airvault (79600)' => '79005', + 'Aix (19200)' => '19002', + 'Aixe-sur-Vienne (87700)' => '87001', + 'Ajain (23380)' => '23002', + 'Ajat (24210)' => '24004', + 'Albignac (19190)' => '19003', + 'Albussac (19380)' => '19004', + 'Alçay-Alçabéhéty-Sunharette (64470)' => '64015', + 'Aldudes (64430)' => '64016', + 'Allas-Bocage (17150)' => '17005', + 'Allas-Champagne (17500)' => '17006', + 'Allas-les-Mines (24220)' => '24006', + 'Allassac (19240)' => '19005', + 'Allemans (24600)' => '24007', + 'Allemans-du-Dropt (47800)' => '47005', + 'Alles-sur-Dordogne (24480)' => '24005', + 'Alleyrat (19200)' => '19006', + 'Alleyrat (23200)' => '23003', + 'Allez-et-Cazeneuve (47110)' => '47006', + 'Allonne (79130)' => '79007', + 'Allons (47420)' => '47007', + 'Alloue (16490)' => '16007', + 'Alos-Sibas-Abense (64470)' => '64017', + 'Altillac (19120)' => '19007', + 'Amailloux (79350)' => '79008', + 'Ambarès-et-Lagrave (33440)' => '33003', + 'Ambazac (87240)' => '87002', + 'Ambérac (16140)' => '16008', + 'Ambernac (16490)' => '16009', + 'Amberre (86110)' => '86002', + 'Ambès (33810)' => '33004', + 'Ambleville (16300)' => '16010', + 'Ambrugeat (19250)' => '19008', + 'Ambrus (47160)' => '47008', + 'Amendeuix-Oneix (64120)' => '64018', + 'Amorots-Succos (64120)' => '64019', + 'Amou (40330)' => '40002', + 'Amuré (79210)' => '79009', + 'Anais (16560)' => '16011', + 'Anais (17540)' => '17007', + 'Ance (64570)' => '64020', + 'Anché (86700)' => '86003', + 'Andernos-les-Bains (33510)' => '33005', + 'Andilly (17230)' => '17008', + 'Andiran (47170)' => '47009', + 'Andoins (64420)' => '64021', + 'Andrein (64390)' => '64022', + 'Angaïs (64510)' => '64023', + 'Angeac-Champagne (16130)' => '16012', + 'Angeac-Charente (16120)' => '16013', + 'Angeduc (16300)' => '16014', + 'Anglade (33390)' => '33006', + 'Angles-sur-l\'Anglin (86260)' => '86004', + 'Anglet (64600)' => '64024', + 'Angliers (17540)' => '17009', + 'Angliers (86330)' => '86005', + 'Angoisse (24270)' => '24008', + 'Angoulême (16000)' => '16015', + 'Angoulins (17690)' => '17010', + 'Angoumé (40990)' => '40003', + 'Angous (64190)' => '64025', + 'Angresse (40150)' => '40004', + 'Anhaux (64220)' => '64026', + 'Anlhiac (24160)' => '24009', + 'Annepont (17350)' => '17011', + 'Annesse-et-Beaulieu (24430)' => '24010', + 'Annezay (17380)' => '17012', + 'Anos (64160)' => '64027', + 'Anoye (64350)' => '64028', + 'Ansac-sur-Vienne (16500)' => '16016', + 'Antagnac (47700)' => '47010', + 'Antezant-la-Chapelle (17400)' => '17013', + 'Anthé (47370)' => '47011', + 'Antigny (86310)' => '86006', + 'Antonne-et-Trigonant (24420)' => '24011', + 'Antran (86100)' => '86007', + 'Anville (16170)' => '16017', + 'Anzême (23000)' => '23004', + 'Anzex (47700)' => '47012', + 'Aramits (64570)' => '64029', + 'Arancou (64270)' => '64031', + 'Araujuzon (64190)' => '64032', + 'Araux (64190)' => '64033', + 'Arbanats (33640)' => '33007', + 'Arbérats-Sillègue (64120)' => '64034', + 'Arbis (33760)' => '33008', + 'Arbonne (64210)' => '64035', + 'Arboucave (40320)' => '40005', + 'Arbouet-Sussaute (64120)' => '64036', + 'Arbus (64230)' => '64037', + 'Arcachon (33120)' => '33009', + 'Arçais (79210)' => '79010', + 'Arcangues (64200)' => '64038', + 'Arçay (86200)' => '86008', + 'Arces (17120)' => '17015', + 'Archiac (17520)' => '17016', + 'Archignac (24590)' => '24012', + 'Archigny (86210)' => '86009', + 'Archingeay (17380)' => '17017', + 'Arcins (33460)' => '33010', + 'Ardilleux (79110)' => '79011', + 'Ardillières (17290)' => '17018', + 'Ardin (79160)' => '79012', + 'Aren (64400)' => '64039', + 'Arengosse (40110)' => '40006', + 'Arès (33740)' => '33011', + 'Aressy (64320)' => '64041', + 'Arette (64570)' => '64040', + 'Arfeuille-Châtain (23700)' => '23005', + 'Argagnon (64300)' => '64042', + 'Argelos (40700)' => '40007', + 'Argelos (64450)' => '64043', + 'Argelouse (40430)' => '40008', + 'Argentat (19400)' => '19010', + 'Argenton (47250)' => '47013', + 'Argenton-l\'Église (79290)' => '79014', + 'Argentonnay (79150)' => '79013', + 'Arget (64410)' => '64044', + 'Arhansus (64120)' => '64045', + 'Arjuzanx (40110)' => '40009', + 'Armendarits (64640)' => '64046', + 'Armillac (47800)' => '47014', + 'Arnac-la-Poste (87160)' => '87003', + 'Arnac-Pompadour (19230)' => '19011', + 'Arnéguy (64220)' => '64047', + 'Arnos (64370)' => '64048', + 'Aroue-Ithorots-Olhaïby (64120)' => '64049', + 'Arrast-Larrebieu (64130)' => '64050', + 'Arraute-Charritte (64120)' => '64051', + 'Arrènes (23210)' => '23006', + 'Arricau-Bordes (64350)' => '64052', + 'Arrien (64420)' => '64053', + 'Arros-de-Nay (64800)' => '64054', + 'Arrosès (64350)' => '64056', + 'Ars (16130)' => '16018', + 'Ars (23480)' => '23007', + 'Ars-en-Ré (17590)' => '17019', + 'Arsac (33460)' => '33012', + 'Arsague (40330)' => '40011', + 'Artassenx (40090)' => '40012', + 'Arthenac (17520)' => '17020', + 'Arthez-d\'Armagnac (40190)' => '40013', + 'Arthez-d\'Asson (64800)' => '64058', + 'Arthez-de-Béarn (64370)' => '64057', + 'Artigueloutan (64420)' => '64059', + 'Artiguelouve (64230)' => '64060', + 'Artigues-près-Bordeaux (33370)' => '33013', + 'Artix (64170)' => '64061', + 'Arudy (64260)' => '64062', + 'Arue (40120)' => '40014', + 'Arvert (17530)' => '17021', + 'Arveyres (33500)' => '33015', + 'Arx (40310)' => '40015', + 'Arzacq-Arraziguet (64410)' => '64063', + 'Asasp-Arros (64660)' => '64064', + 'Ascain (64310)' => '64065', + 'Ascarat (64220)' => '64066', + 'Aslonnes (86340)' => '86010', + 'Asnières-en-Poitou (79170)' => '79015', + 'Asnières-la-Giraud (17400)' => '17022', + 'Asnières-sur-Blour (86430)' => '86011', + 'Asnières-sur-Nouère (16290)' => '16019', + 'Asnois (86250)' => '86012', + 'Asques (33240)' => '33016', + 'Assais-les-Jumeaux (79600)' => '79016', + 'Assat (64510)' => '64067', + 'Asson (64800)' => '64068', + 'Astaffort (47220)' => '47015', + 'Astaillac (19120)' => '19012', + 'Aste-Béon (64260)' => '64069', + 'Astis (64450)' => '64070', + 'Athos-Aspis (64390)' => '64071', + 'Aubagnan (40700)' => '40016', + 'Aubas (24290)' => '24014', + 'Aubazines (19190)' => '19013', + 'Aubertin (64290)' => '64072', + 'Aubeterre-sur-Dronne (16390)' => '16020', + 'Aubiac (33430)' => '33017', + 'Aubiac (47310)' => '47016', + 'Aubigné (79110)' => '79018', + 'Aubigny (79390)' => '79019', + 'Aubin (64230)' => '64073', + 'Aubous (64330)' => '64074', + 'Aubusson (23200)' => '23008', + 'Audaux (64190)' => '64075', + 'Audenge (33980)' => '33019', + 'Audignon (40500)' => '40017', + 'Audon (40400)' => '40018', + 'Audrix (24260)' => '24015', + 'Auga (64450)' => '64077', + 'Auge (23170)' => '23009', + 'Augé (79400)' => '79020', + 'Auge-Saint-Médard (16170)' => '16339', + 'Augères (23210)' => '23010', + 'Augignac (24300)' => '24016', + 'Augne (87120)' => '87004', + 'Aujac (17770)' => '17023', + 'Aulnay (17470)' => '17024', + 'Aulnay (86330)' => '86013', + 'Aulon (23210)' => '23011', + 'Aumagne (17770)' => '17025', + 'Aunac (16460)' => '16023', + 'Auradou (47140)' => '47017', + 'Aureil (87220)' => '87005', + 'Aureilhan (40200)' => '40019', + 'Auriac (19220)' => '19014', + 'Auriac (64450)' => '64078', + 'Auriac-du-Périgord (24290)' => '24018', + 'Auriac-sur-Dropt (47120)' => '47018', + 'Auriat (23400)' => '23012', + 'Aurice (40500)' => '40020', + 'Auriolles (33790)' => '33020', + 'Aurions-Idernes (64350)' => '64079', + 'Auros (33124)' => '33021', + 'Aussac-Vadalle (16560)' => '16024', + 'Aussevielle (64230)' => '64080', + 'Aussurucq (64130)' => '64081', + 'Auterrive (64270)' => '64082', + 'Autevielle-Saint-Martin-Bideren (64390)' => '64083', + 'Authon-Ébéon (17770)' => '17026', + 'Auzances (23700)' => '23013', + 'Availles-en-Châtellerault (86530)' => '86014', + 'Availles-Limouzine (86460)' => '86015', + 'Availles-Thouarsais (79600)' => '79022', + 'Avanton (86170)' => '86016', + 'Avensan (33480)' => '33022', + 'Avon (79800)' => '79023', + 'Avy (17800)' => '17027', + 'Aydie (64330)' => '64084', + 'Aydius (64490)' => '64085', + 'Ayen (19310)' => '19015', + 'Ayguemorte-les-Graves (33640)' => '33023', + 'Ayherre (64240)' => '64086', + 'Ayron (86190)' => '86017', + 'Aytré (17440)' => '17028', + 'Azat-Châtenet (23210)' => '23014', + 'Azat-le-Ris (87360)' => '87006', + 'Azay-le-Brûlé (79400)' => '79024', + 'Azay-sur-Thouet (79130)' => '79025', + 'Azerables (23160)' => '23015', + 'Azerat (24210)' => '24019', + 'Azur (40140)' => '40021', + 'Badefols-d\'Ans (24390)' => '24021', + 'Badefols-sur-Dordogne (24150)' => '24022', + 'Bagas (33190)' => '33024', + 'Bagnizeau (17160)' => '17029', + 'Bahus-Soubiran (40320)' => '40022', + 'Baigneaux (33760)' => '33025', + 'Baignes-Sainte-Radegonde (16360)' => '16025', + 'Baigts (40380)' => '40023', + 'Baigts-de-Béarn (64300)' => '64087', + 'Bajamont (47480)' => '47019', + 'Balansun (64300)' => '64088', + 'Balanzac (17600)' => '17030', + 'Baleix (64460)' => '64089', + 'Baleyssagues (47120)' => '47020', + 'Baliracq-Maumusson (64330)' => '64090', + 'Baliros (64510)' => '64091', + 'Balizac (33730)' => '33026', + 'Ballans (17160)' => '17031', + 'Balledent (87290)' => '87007', + 'Ballon (17290)' => '17032', + 'Balzac (16430)' => '16026', + 'Banca (64430)' => '64092', + 'Baneuil (24150)' => '24023', + 'Banize (23120)' => '23016', + 'Banos (40500)' => '40024', + 'Bar (19800)' => '19016', + 'Barbaste (47230)' => '47021', + 'Barbezières (16140)' => '16027', + 'Barbezieux-Saint-Hilaire (16300)' => '16028', + 'Barcus (64130)' => '64093', + 'Bardenac (16210)' => '16029', + 'Bardos (64520)' => '64094', + 'Bardou (24560)' => '24024', + 'Barie (33190)' => '33027', + 'Barinque (64160)' => '64095', + 'Baron (33750)' => '33028', + 'Barraute-Camu (64390)' => '64096', + 'Barret (16300)' => '16030', + 'Barro (16700)' => '16031', + 'Bars (24210)' => '24025', + 'Barsac (33720)' => '33030', + 'Barzan (17120)' => '17034', + 'Barzun (64530)' => '64097', + 'Bas-Mauco (40500)' => '40026', + 'Bascons (40090)' => '40025', + 'Bassac (16120)' => '16032', + 'Bassanne (33190)' => '33031', + 'Bassens (33530)' => '33032', + 'Bassercles (40700)' => '40027', + 'Basses (86200)' => '86018', + 'Bassignac-le-Bas (19430)' => '19017', + 'Bassignac-le-Haut (19220)' => '19018', + 'Bassillac (24330)' => '24026', + 'Bassillon-Vauzé (64350)' => '64098', + 'Bassussarry (64200)' => '64100', + 'Bastanès (64190)' => '64099', + 'Bastennes (40360)' => '40028', + 'Basville (23260)' => '23017', + 'Bats (40320)' => '40029', + 'Baudignan (40310)' => '40030', + 'Baudreix (64800)' => '64101', + 'Baurech (33880)' => '33033', + 'Bayac (24150)' => '24027', + 'Bayas (33230)' => '33034', + 'Bayers (16460)' => '16033', + 'Bayon-sur-Gironde (33710)' => '33035', + 'Bayonne (64100)' => '64102', + 'Bazac (16210)' => '16034', + 'Bazas (33430)' => '33036', + 'Bazauges (17490)' => '17035', + 'Bazelat (23160)' => '23018', + 'Bazens (47130)' => '47022', + 'Beaugas (47290)' => '47023', + 'Beaugeay (17620)' => '17036', + 'Beaulieu-sous-Parthenay (79420)' => '79029', + 'Beaulieu-sur-Dordogne (19120)' => '19019', + 'Beaulieu-sur-Sonnette (16450)' => '16035', + 'Beaumont (19390)' => '19020', + 'Beaumont (86490)' => '86019', + 'Beaumont-du-Lac (87120)' => '87009', + 'Beaumontois en Périgord (24440)' => '24028', + 'Beaupouyet (24400)' => '24029', + 'Beaupuy (47200)' => '47024', + 'Beauregard-de-Terrasson (24120)' => '24030', + 'Beauregard-et-Bassac (24140)' => '24031', + 'Beauronne (24400)' => '24032', + 'Beaussac (24340)' => '24033', + 'Beaussais-Vitré (79370)' => '79030', + 'Beautiran (33640)' => '33037', + 'Beauvais-sur-Matha (17490)' => '17037', + 'Beauville (47470)' => '47025', + 'Beauvoir-sur-Niort (79360)' => '79031', + 'Beauziac (47700)' => '47026', + 'Béceleuf (79160)' => '79032', + 'Bécheresse (16250)' => '16036', + 'Bédeille (64460)' => '64103', + 'Bedenac (17210)' => '17038', + 'Bedous (64490)' => '64104', + 'Bégaar (40400)' => '40031', + 'Bégadan (33340)' => '33038', + 'Bègles (33130)' => '33039', + 'Béguey (33410)' => '33040', + 'Béguios (64120)' => '64105', + 'Béhasque-Lapiste (64120)' => '64106', + 'Béhorléguy (64220)' => '64107', + 'Beissat (23260)' => '23019', + 'Beleymas (24140)' => '24034', + 'Belhade (40410)' => '40032', + 'Belin-Béliet (33830)' => '33042', + 'Bélis (40120)' => '40033', + 'Bellac (87300)' => '87011', + 'Bellebat (33760)' => '33043', + 'Bellechassagne (19290)' => '19021', + 'Bellefond (33760)' => '33044', + 'Bellefonds (86210)' => '86020', + 'Bellegarde-en-Marche (23190)' => '23020', + 'Belleville (79360)' => '79033', + 'Bellocq (64270)' => '64108', + 'Bellon (16210)' => '16037', + 'Belluire (17800)' => '17039', + 'Bélus (40300)' => '40034', + 'Belvès-de-Castillon (33350)' => '33045', + 'Benassay (86470)' => '86021', + 'Benayes (19510)' => '19022', + 'Bénéjacq (64800)' => '64109', + 'Bénesse-lès-Dax (40180)' => '40035', + 'Bénesse-Maremne (40230)' => '40036', + 'Benest (16350)' => '16038', + 'Bénévent-l\'Abbaye (23210)' => '23021', + 'Benon (17170)' => '17041', + 'Benquet (40280)' => '40037', + 'Bentayou-Sérée (64460)' => '64111', + 'Béost (64440)' => '64110', + 'Berbiguières (24220)' => '24036', + 'Bercloux (17770)' => '17042', + 'Bérenx (64300)' => '64112', + 'Bergerac (24100)' => '24037', + 'Bergouey (40250)' => '40038', + 'Bergouey-Viellenave (64270)' => '64113', + 'Bernac (16700)' => '16039', + 'Bernadets (64160)' => '64114', + 'Bernay-Saint-Martin (17330)' => '17043', + 'Berneuil (16480)' => '16040', + 'Berneuil (17460)' => '17044', + 'Berneuil (87300)' => '87012', + 'Bernos-Beaulac (33430)' => '33046', + 'Berrie (86120)' => '86022', + 'Berrogain-Laruns (64130)' => '64115', + 'Bersac-sur-Rivalier (87370)' => '87013', + 'Berson (33390)' => '33047', + 'Berthegon (86420)' => '86023', + 'Berthez (33124)' => '33048', + 'Bertric-Burée (24320)' => '24038', + 'Béruges (86190)' => '86024', + 'Bescat (64260)' => '64116', + 'Bésingrand (64150)' => '64117', + 'Bessac (16250)' => '16041', + 'Bessé (16140)' => '16042', + 'Besse (24550)' => '24039', + 'Bessines (79000)' => '79034', + 'Bessines-sur-Gartempe (87250)' => '87014', + 'Betbezer-d\'Armagnac (40240)' => '40039', + 'Bétête (23270)' => '23022', + 'Béthines (86310)' => '86025', + 'Bétracq (64350)' => '64118', + 'Beurlay (17250)' => '17045', + 'Beuste (64800)' => '64119', + 'Beuxes (86120)' => '86026', + 'Beychac-et-Caillau (33750)' => '33049', + 'Beylongue (40370)' => '40040', + 'Beynac (87700)' => '87015', + 'Beynac-et-Cazenac (24220)' => '24040', + 'Beynat (19190)' => '19023', + 'Beyrie-en-Béarn (64230)' => '64121', + 'Beyrie-sur-Joyeuse (64120)' => '64120', + 'Beyries (40700)' => '40041', + 'Beyssac (19230)' => '19024', + 'Beyssenac (19230)' => '19025', + 'Bézenac (24220)' => '24041', + 'Biard (86580)' => '86027', + 'Biarritz (64200)' => '64122', + 'Biarrotte (40390)' => '40042', + 'Bias (40170)' => '40043', + 'Bias (47300)' => '47027', + 'Biaudos (40390)' => '40044', + 'Bidache (64520)' => '64123', + 'Bidarray (64780)' => '64124', + 'Bidart (64210)' => '64125', + 'Bidos (64400)' => '64126', + 'Bielle (64260)' => '64127', + 'Bieujac (33210)' => '33050', + 'Biganos (33380)' => '33051', + 'Bignay (17400)' => '17046', + 'Bignoux (86800)' => '86028', + 'Bilhac (19120)' => '19026', + 'Bilhères (64260)' => '64128', + 'Billère (64140)' => '64129', + 'Bioussac (16700)' => '16044', + 'Birac (16120)' => '16045', + 'Birac (33430)' => '33053', + 'Birac-sur-Trec (47200)' => '47028', + 'Biras (24310)' => '24042', + 'Biriatou (64700)' => '64130', + 'Biron (17800)' => '17047', + 'Biron (24540)' => '24043', + 'Biron (64300)' => '64131', + 'Biscarrosse (40600)' => '40046', + 'Bizanos (64320)' => '64132', + 'Blaignac (33190)' => '33054', + 'Blaignan (33340)' => '33055', + 'Blanquefort (33290)' => '33056', + 'Blanquefort-sur-Briolance (47500)' => '47029', + 'Blanzac (87300)' => '87017', + 'Blanzac-lès-Matha (17160)' => '17048', + 'Blanzac-Porcheresse (16250)' => '16046', + 'Blanzaguet-Saint-Cybard (16320)' => '16047', + 'Blanzay (86400)' => '86029', + 'Blanzay-sur-Boutonne (17470)' => '17049', + 'Blasimon (33540)' => '33057', + 'Blaslay (86170)' => '86030', + 'Blaudeix (23140)' => '23023', + 'Blaye (33390)' => '33058', + 'Blaymont (47470)' => '47030', + 'Blésignac (33670)' => '33059', + 'Blessac (23200)' => '23024', + 'Blis-et-Born (24330)' => '24044', + 'Blond (87300)' => '87018', + 'Boé (47550)' => '47031', + 'Boeil-Bezing (64510)' => '64133', + 'Bois (17240)' => '17050', + 'Boisbreteau (16480)' => '16048', + 'Boismé (79300)' => '79038', + 'Boisné-La Tude (16320)' => '16082', + 'Boisredon (17150)' => '17052', + 'Boisse (24560)' => '24045', + 'Boisserolles (79360)' => '79039', + 'Boisseuil (87220)' => '87019', + 'Boisseuilh (24390)' => '24046', + 'Bommes (33210)' => '33060', + 'Bon-Encontre (47240)' => '47032', + 'Bonloc (64240)' => '64134', + 'Bonnac-la-Côte (87270)' => '87020', + 'Bonnat (23220)' => '23025', + 'Bonnefond (19170)' => '19027', + 'Bonnegarde (40330)' => '40047', + 'Bonnes (16390)' => '16049', + 'Bonnes (86300)' => '86031', + 'Bonnetan (33370)' => '33061', + 'Bonneuil (16120)' => '16050', + 'Bonneuil-Matours (86210)' => '86032', + 'Bonneville (16170)' => '16051', + 'Bonneville-et-Saint-Avit-de-Fumadières (24230)' => '24048', + 'Bonnut (64300)' => '64135', + 'Bonzac (33910)' => '33062', + 'Boos (40370)' => '40048', + 'Borce (64490)' => '64136', + 'Bord-Saint-Georges (23230)' => '23026', + 'Bordeaux (33000)' => '33063', + 'Bordères (64800)' => '64137', + 'Bordères-et-Lamensans (40270)' => '40049', + 'Bordes (64510)' => '64138', + 'Bords (17430)' => '17053', + 'Boresse-et-Martron (17270)' => '17054', + 'Borrèze (24590)' => '24050', + 'Bors (Canton de Baignes-Sainte-Radegonde) (16360)' => '16053', + 'Bors (Canton de Montmoreau-Saint-Cybard) (16190)' => '16052', + 'Bort-les-Orgues (19110)' => '19028', + 'Boscamnant (17360)' => '17055', + 'Bosdarros (64290)' => '64139', + 'Bosmie-l\'Aiguille (87110)' => '87021', + 'Bosmoreau-les-Mines (23400)' => '23027', + 'Bosroger (23200)' => '23028', + 'Bosset (24130)' => '24051', + 'Bossugan (33350)' => '33064', + 'Bostens (40090)' => '40050', + 'Boucau (64340)' => '64140', + 'Boudy-de-Beauregard (47290)' => '47033', + 'Boueilh-Boueilho-Lasque (64330)' => '64141', + 'Bouëx (16410)' => '16055', + 'Bougarber (64230)' => '64142', + 'Bouglon (47250)' => '47034', + 'Bougneau (17800)' => '17056', + 'Bougon (79800)' => '79042', + 'Bougue (40090)' => '40051', + 'Bouhet (17540)' => '17057', + 'Bouillac (24480)' => '24052', + 'Bouillé-Loretz (79290)' => '79043', + 'Bouillé-Saint-Paul (79290)' => '79044', + 'Bouillon (64410)' => '64143', + 'Bouin (79110)' => '79045', + 'Boulazac Isle Manoire (24750)' => '24053', + 'Bouliac (33270)' => '33065', + 'Boumourt (64370)' => '64144', + 'Bouniagues (24560)' => '24054', + 'Bourcefranc-le-Chapus (17560)' => '17058', + 'Bourdalat (40190)' => '40052', + 'Bourdeilles (24310)' => '24055', + 'Bourdelles (33190)' => '33066', + 'Bourdettes (64800)' => '64145', + 'Bouresse (86410)' => '86034', + 'Bourg (33710)' => '33067', + 'Bourg-Archambault (86390)' => '86035', + 'Bourg-Charente (16200)' => '16056', + 'Bourg-des-Maisons (24320)' => '24057', + 'Bourg-du-Bost (24600)' => '24058', + 'Bourganeuf (23400)' => '23030', + 'Bourgnac (24400)' => '24059', + 'Bourgneuf (17220)' => '17059', + 'Bourgougnague (47410)' => '47035', + 'Bourideys (33113)' => '33068', + 'Bourlens (47370)' => '47036', + 'Bournand (86120)' => '86036', + 'Bournel (47210)' => '47037', + 'Bourniquel (24150)' => '24060', + 'Bournos (64450)' => '64146', + 'Bourran (47320)' => '47038', + 'Bourriot-Bergonce (40120)' => '40053', + 'Bourrou (24110)' => '24061', + 'Boussac (23600)' => '23031', + 'Boussac-Bourg (23600)' => '23032', + 'Boussais (79600)' => '79047', + 'Boussès (47420)' => '47039', + 'Bouteilles-Saint-Sébastien (24320)' => '24062', + 'Boutenac-Touvent (17120)' => '17060', + 'Bouteville (16120)' => '16057', + 'Boutiers-Saint-Trojan (16100)' => '16058', + 'Bouzic (24250)' => '24063', + 'Brach (33480)' => '33070', + 'Bran (17210)' => '17061', + 'Branceilles (19500)' => '19029', + 'Branne (33420)' => '33071', + 'Brannens (33124)' => '33072', + 'Brantôme en Périgord (24310)' => '24064', + 'Brassempouy (40330)' => '40054', + 'Braud-et-Saint-Louis (33820)' => '33073', + 'Brax (47310)' => '47040', + 'Bresdon (17490)' => '17062', + 'Bressuire (79300)' => '79049', + 'Bretagne-de-Marsan (40280)' => '40055', + 'Bretignolles (79140)' => '79050', + 'Brettes (16240)' => '16059', + 'Breuil-la-Réorte (17700)' => '17063', + 'Breuil-Magné (17870)' => '17065', + 'Breuilaufa (87300)' => '87022', + 'Breuilh (24380)' => '24065', + 'Breuillet (17920)' => '17064', + 'Bréville (16370)' => '16060', + 'Brie (16590)' => '16061', + 'Brie (79100)' => '79054', + 'Brie-sous-Archiac (17520)' => '17066', + 'Brie-sous-Barbezieux (16300)' => '16062', + 'Brie-sous-Chalais (16210)' => '16063', + 'Brie-sous-Matha (17160)' => '17067', + 'Brie-sous-Mortagne (17120)' => '17068', + 'Brieuil-sur-Chizé (79170)' => '79055', + 'Brignac-la-Plaine (19310)' => '19030', + 'Brigueil-le-Chantre (86290)' => '86037', + 'Brigueuil (16420)' => '16064', + 'Brillac (16500)' => '16065', + 'Brion (86160)' => '86038', + 'Brion-près-Thouet (79290)' => '79056', + 'Brioux-sur-Boutonne (79170)' => '79057', + 'Briscous (64240)' => '64147', + 'Brive-la-Gaillarde (19100)' => '19031', + 'Brives-sur-Charente (17800)' => '17069', + 'Brivezac (19120)' => '19032', + 'Brizambourg (17770)' => '17070', + 'Brocas (40420)' => '40056', + 'Brossac (16480)' => '16066', + 'Brouchaud (24210)' => '24066', + 'Brouqueyran (33124)' => '33074', + 'Brousse (23700)' => '23034', + 'Bruch (47130)' => '47041', + 'Bruges (33520)' => '33075', + 'Bruges-Capbis-Mifaget (64800)' => '64148', + 'Brugnac (47260)' => '47042', + 'Brûlain (79230)' => '79058', + 'Brux (86510)' => '86039', + 'Buanes (40320)' => '40057', + 'Budelière (23170)' => '23035', + 'Budos (33720)' => '33076', + 'Bugeat (19170)' => '19033', + 'Bugnein (64190)' => '64149', + 'Bujaleuf (87460)' => '87024', + 'Bunus (64120)' => '64150', + 'Bunzac (16110)' => '16067', + 'Burgaronne (64390)' => '64151', + 'Burgnac (87800)' => '87025', + 'Burie (17770)' => '17072', + 'Buros (64160)' => '64152', + 'Burosse-Mendousse (64330)' => '64153', + 'Bussac (24350)' => '24069', + 'Bussac-Forêt (17210)' => '17074', + 'Bussac-sur-Charente (17100)' => '17073', + 'Busserolles (24360)' => '24070', + 'Bussière-Badil (24360)' => '24071', + 'Bussière-Dunoise (23320)' => '23036', + 'Bussière-Galant (87230)' => '87027', + 'Bussière-Nouvelle (23700)' => '23037', + 'Bussière-Poitevine (87320)' => '87028', + 'Bussière-Saint-Georges (23600)' => '23038', + 'Bussunarits-Sarrasquette (64220)' => '64154', + 'Bustince-Iriberry (64220)' => '64155', + 'Buxerolles (86180)' => '86041', + 'Buxeuil (37160)' => '86042', + 'Buzet-sur-Baïse (47160)' => '47043', + 'Buziet (64680)' => '64156', + 'Buzy (64260)' => '64157', + 'Cabanac-et-Villagrains (33650)' => '33077', + 'Cabara (33420)' => '33078', + 'Cabariot (17430)' => '17075', + 'Cabidos (64410)' => '64158', + 'Cachen (40120)' => '40058', + 'Cadarsac (33750)' => '33079', + 'Cadaujac (33140)' => '33080', + 'Cadillac (33410)' => '33081', + 'Cadillac-en-Fronsadais (33240)' => '33082', + 'Cadillon (64330)' => '64159', + 'Cagnotte (40300)' => '40059', + 'Cahuzac (47330)' => '47044', + 'Calès (24150)' => '24073', + 'Calignac (47600)' => '47045', + 'Callen (40430)' => '40060', + 'Calonges (47430)' => '47046', + 'Calviac-en-Périgord (24370)' => '24074', + 'Camarsac (33750)' => '33083', + 'Cambes (33880)' => '33084', + 'Cambes (47350)' => '47047', + 'Camblanes-et-Meynac (33360)' => '33085', + 'Cambo-les-Bains (64250)' => '64160', + 'Came (64520)' => '64161', + 'Camiac-et-Saint-Denis (33420)' => '33086', + 'Camiran (33190)' => '33087', + 'Camou-Cihigue (64470)' => '64162', + 'Campagnac-lès-Quercy (24550)' => '24075', + 'Campagne (24260)' => '24076', + 'Campagne (40090)' => '40061', + 'Campet-et-Lamolère (40090)' => '40062', + 'Camps-Saint-Mathurin-Léobazel (19430)' => '19034', + 'Camps-sur-l\'Isle (33660)' => '33088', + 'Campsegret (24140)' => '24077', + 'Campugnan (33390)' => '33089', + 'Cancon (47290)' => '47048', + 'Candresse (40180)' => '40063', + 'Canéjan (33610)' => '33090', + 'Canenx-et-Réaut (40090)' => '40064', + 'Cantenac (33460)' => '33091', + 'Cantillac (24530)' => '24079', + 'Cantois (33760)' => '33092', + 'Capbreton (40130)' => '40065', + 'Capdrot (24540)' => '24080', + 'Capian (33550)' => '33093', + 'Caplong (33220)' => '33094', + 'Captieux (33840)' => '33095', + 'Carbon-Blanc (33560)' => '33096', + 'Carcans (33121)' => '33097', + 'Carcarès-Sainte-Croix (40400)' => '40066', + 'Carcen-Ponson (40400)' => '40067', + 'Cardan (33410)' => '33098', + 'Cardesse (64360)' => '64165', + 'Carignan-de-Bordeaux (33360)' => '33099', + 'Carlux (24370)' => '24081', + 'Caro (64220)' => '64166', + 'Carrère (64160)' => '64167', + 'Carresse-Cassaber (64270)' => '64168', + 'Cars (33390)' => '33100', + 'Carsac-Aillac (24200)' => '24082', + 'Carsac-de-Gurson (24610)' => '24083', + 'Cartelègue (33390)' => '33101', + 'Carves (24170)' => '24084', + 'Cassen (40380)' => '40068', + 'Casseneuil (47440)' => '47049', + 'Casseuil (33190)' => '33102', + 'Cassignas (47340)' => '47050', + 'Castagnède (64270)' => '64170', + 'Castaignos-Souslens (40700)' => '40069', + 'Castandet (40270)' => '40070', + 'Casteide-Cami (64170)' => '64171', + 'Casteide-Candau (64370)' => '64172', + 'Casteide-Doat (64460)' => '64173', + 'Castel-Sarrazin (40330)' => '40074', + 'Castelculier (47240)' => '47051', + 'Casteljaloux (47700)' => '47052', + 'Castella (47340)' => '47053', + 'Castelmoron-d\'Albret (33540)' => '33103', + 'Castelmoron-sur-Lot (47260)' => '47054', + 'Castelnau-Chalosse (40360)' => '40071', + 'Castelnau-de-Médoc (33480)' => '33104', + 'Castelnau-sur-Gupie (47180)' => '47056', + 'Castelnau-Tursan (40320)' => '40072', + 'Castelnaud-de-Gratecambe (47290)' => '47055', + 'Castelnaud-la-Chapelle (24250)' => '24086', + 'Castelner (40700)' => '40073', + 'Castels (24220)' => '24087', + 'Castelviel (33540)' => '33105', + 'Castéra-Loubix (64460)' => '64174', + 'Castet (64260)' => '64175', + 'Castetbon (64190)' => '64176', + 'Castétis (64300)' => '64177', + 'Castetnau-Camblong (64190)' => '64178', + 'Castetner (64300)' => '64179', + 'Castetpugon (64330)' => '64180', + 'Castets (40260)' => '40075', + 'Castets-en-Dorthe (33210)' => '33106', + 'Castillon (Canton d\'Arthez-de-Béarn) (64370)' => '64181', + 'Castillon (Canton de Lembeye) (64350)' => '64182', + 'Castillon-de-Castets (33210)' => '33107', + 'Castillon-la-Bataille (33350)' => '33108', + 'Castillonnès (47330)' => '47057', + 'Castres-Gironde (33640)' => '33109', + 'Caubeyres (47160)' => '47058', + 'Caubios-Loos (64230)' => '64183', + 'Caubon-Saint-Sauveur (47120)' => '47059', + 'Caudecoste (47220)' => '47060', + 'Caudrot (33490)' => '33111', + 'Caumont (33540)' => '33112', + 'Caumont-sur-Garonne (47430)' => '47061', + 'Cauna (40500)' => '40076', + 'Caunay (79190)' => '79060', + 'Cauneille (40300)' => '40077', + 'Caupenne (40250)' => '40078', + 'Cause-de-Clérans (24150)' => '24088', + 'Cauvignac (33690)' => '33113', + 'Cauzac (47470)' => '47062', + 'Cavarc (47330)' => '47063', + 'Cavignac (33620)' => '33114', + 'Cazalis (33113)' => '33115', + 'Cazalis (40700)' => '40079', + 'Cazats (33430)' => '33116', + 'Cazaugitat (33790)' => '33117', + 'Cazères-sur-l\'Adour (40270)' => '40080', + 'Cazideroque (47370)' => '47064', + 'Cazoulès (24370)' => '24089', + 'Ceaux-en-Couhé (86700)' => '86043', + 'Ceaux-en-Loudun (86200)' => '86044', + 'Celle-Lévescault (86600)' => '86045', + 'Cellefrouin (16260)' => '16068', + 'Celles (17520)' => '17076', + 'Celles (24600)' => '24090', + 'Celles-sur-Belle (79370)' => '79061', + 'Cellettes (16230)' => '16069', + 'Cénac (33360)' => '33118', + 'Cénac-et-Saint-Julien (24250)' => '24091', + 'Cendrieux (24380)' => '24092', + 'Cenon (33150)' => '33119', + 'Cenon-sur-Vienne (86530)' => '86046', + 'Cercles (24320)' => '24093', + 'Cercoux (17270)' => '17077', + 'Cère (40090)' => '40081', + 'Cerizay (79140)' => '79062', + 'Cernay (86140)' => '86047', + 'Cérons (33720)' => '33120', + 'Cersay (79290)' => '79063', + 'Cescau (64170)' => '64184', + 'Cessac (33760)' => '33121', + 'Cestas (33610)' => '33122', + 'Cette-Eygun (64490)' => '64185', + 'Ceyroux (23210)' => '23042', + 'Cézac (33620)' => '33123', + 'Chabanais (16150)' => '16070', + 'Chabournay (86380)' => '86048', + 'Chabrac (16150)' => '16071', + 'Chabrignac (19350)' => '19035', + 'Chadenac (17800)' => '17078', + 'Chadurie (16250)' => '16072', + 'Chail (79500)' => '79064', + 'Chaillac-sur-Vienne (87200)' => '87030', + 'Chaillevette (17890)' => '17079', + 'Chalagnac (24380)' => '24094', + 'Chalais (16210)' => '16073', + 'Chalais (24800)' => '24095', + 'Chalais (86200)' => '86049', + 'Chalandray (86190)' => '86050', + 'Challignac (16300)' => '16074', + 'Châlus (87230)' => '87032', + 'Chamadelle (33230)' => '33124', + 'Chamberaud (23480)' => '23043', + 'Chamberet (19370)' => '19036', + 'Chambon (17290)' => '17080', + 'Chambon-Sainte-Croix (23220)' => '23044', + 'Chambon-sur-Voueize (23170)' => '23045', + 'Chambonchard (23110)' => '23046', + 'Chamborand (23240)' => '23047', + 'Chamboret (87140)' => '87033', + 'Chamboulive (19450)' => '19037', + 'Chameyrat (19330)' => '19038', + 'Chamouillac (17130)' => '17081', + 'Champagnac (17500)' => '17082', + 'Champagnac-de-Belair (24530)' => '24096', + 'Champagnac-la-Noaille (19320)' => '19039', + 'Champagnac-la-Prune (19320)' => '19040', + 'Champagnac-la-Rivière (87150)' => '87034', + 'Champagnat (23190)' => '23048', + 'Champagne (17620)' => '17083', + 'Champagne-et-Fontaine (24320)' => '24097', + 'Champagné-le-Sec (86510)' => '86051', + 'Champagne-Mouton (16350)' => '16076', + 'Champagné-Saint-Hilaire (86160)' => '86052', + 'Champagne-Vigny (16250)' => '16075', + 'Champagnolles (17240)' => '17084', + 'Champcevinel (24750)' => '24098', + 'Champdeniers-Saint-Denis (79220)' => '79066', + 'Champdolent (17430)' => '17085', + 'Champeaux-et-la-Chapelle-Pommier (24340)' => '24099', + 'Champigny-le-Sec (86170)' => '86053', + 'Champmillon (16290)' => '16077', + 'Champnétery (87400)' => '87035', + 'Champniers (16430)' => '16078', + 'Champniers (86400)' => '86054', + 'Champniers-et-Reilhac (24360)' => '24100', + 'Champs-Romain (24470)' => '24101', + 'Champsac (87230)' => '87036', + 'Champsanglard (23220)' => '23049', + 'Chanac-les-Mines (19150)' => '19041', + 'Chancelade (24650)' => '24102', + 'Chaniers (17610)' => '17086', + 'Chantecorps (79340)' => '79068', + 'Chanteix (19330)' => '19042', + 'Chanteloup (79320)' => '79069', + 'Chantemerle-sur-la-Soie (17380)' => '17087', + 'Chantérac (24190)' => '24104', + 'Chantillac (16360)' => '16079', + 'Chapdeuil (24320)' => '24105', + 'Chapelle-Spinasse (19300)' => '19046', + 'Chapelle-Viviers (86300)' => '86059', + 'Chaptelat (87270)' => '87038', + 'Chard (23700)' => '23053', + 'Charmé (16140)' => '16083', + 'Charrais (86170)' => '86060', + 'Charras (16380)' => '16084', + 'Charre (64190)' => '64186', + 'Charritte-de-Bas (64130)' => '64187', + 'Charron (17230)' => '17091', + 'Charron (23700)' => '23054', + 'Charroux (86250)' => '86061', + 'Chartrier-Ferrière (19600)' => '19047', + 'Chartuzac (17130)' => '17092', + 'Chassaignes (24600)' => '24114', + 'Chasseneuil-du-Poitou (86360)' => '86062', + 'Chasseneuil-sur-Bonnieure (16260)' => '16085', + 'Chassenon (16150)' => '16086', + 'Chassiecq (16350)' => '16087', + 'Chassors (16200)' => '16088', + 'Chasteaux (19600)' => '19049', + 'Chatain (86250)' => '86063', + 'Château-Chervix (87380)' => '87039', + 'Château-Garnier (86350)' => '86064', + 'Château-l\'Évêque (24460)' => '24115', + 'Château-Larcher (86370)' => '86065', + 'Châteaubernard (16100)' => '16089', + 'Châteauneuf-la-Forêt (87130)' => '87040', + 'Châteauneuf-sur-Charente (16120)' => '16090', + 'Châteauponsac (87290)' => '87041', + 'Châtelaillon-Plage (17340)' => '17094', + 'Châtelard (23700)' => '23055', + 'Châtellerault (86100)' => '86066', + 'Châtelus-le-Marcheix (23430)' => '23056', + 'Châtelus-Malvaleix (23270)' => '23057', + 'Chatenet (17210)' => '17095', + 'Châtignac (16480)' => '16091', + 'Châtillon (86700)' => '86067', + 'Châtillon-sur-Thouet (79200)' => '79080', + 'Châtres (24120)' => '24116', + 'Chauffour-sur-Vell (19500)' => '19050', + 'Chaumeil (19390)' => '19051', + 'Chaunac (17130)' => '17096', + 'Chaunay (86510)' => '86068', + 'Chauray (79180)' => '79081', + 'Chauvigny (86300)' => '86070', + 'Chavagnac (24120)' => '24117', + 'Chavanac (19290)' => '19052', + 'Chavanat (23250)' => '23060', + 'Chaveroche (19200)' => '19053', + 'Chazelles (16380)' => '16093', + 'Chef-Boutonne (79110)' => '79083', + 'Cheissoux (87460)' => '87043', + 'Chenac-Saint-Seurin-d\'Uzet (17120)' => '17098', + 'Chenailler-Mascheix (19120)' => '19054', + 'Chenay (79120)' => '79084', + 'Cheneché (86380)' => '86071', + 'Chénérailles (23130)' => '23061', + 'Chenevelles (86450)' => '86072', + 'Chéniers (23220)' => '23062', + 'Chenommet (16460)' => '16094', + 'Chenon (16460)' => '16095', + 'Chepniers (17210)' => '17099', + 'Chérac (17610)' => '17100', + 'Chéraute (64130)' => '64188', + 'Cherbonnières (17470)' => '17101', + 'Chérigné (79170)' => '79085', + 'Chermignac (17460)' => '17102', + 'Chéronnac (87600)' => '87044', + 'Cherval (24320)' => '24119', + 'Cherveix-Cubas (24390)' => '24120', + 'Cherves (86170)' => '86073', + 'Cherves-Châtelars (16310)' => '16096', + 'Cherves-Richemont (16370)' => '16097', + 'Chervettes (17380)' => '17103', + 'Cherveux (79410)' => '79086', + 'Chevanceaux (17210)' => '17104', + 'Chey (79120)' => '79087', + 'Chiché (79350)' => '79088', + 'Chillac (16480)' => '16099', + 'Chirac (16150)' => '16100', + 'Chirac-Bellevue (19160)' => '19055', + 'Chiré-en-Montreuil (86190)' => '86074', + 'Chives (17510)' => '17105', + 'Chizé (79170)' => '79090', + 'Chouppes (86110)' => '86075', + 'Chourgnac (24640)' => '24121', + 'Ciboure (64500)' => '64189', + 'Cierzac (17520)' => '17106', + 'Cieux (87520)' => '87045', + 'Ciré-d\'Aunis (17290)' => '17107', + 'Cirières (79140)' => '79091', + 'Cissac-Médoc (33250)' => '33125', + 'Cissé (86170)' => '86076', + 'Civaux (86320)' => '86077', + 'Civrac-de-Blaye (33920)' => '33126', + 'Civrac-en-Médoc (33340)' => '33128', + 'Civrac-sur-Dordogne (33350)' => '33127', + 'Civray (86400)' => '86078', + 'Cladech (24170)' => '24122', + 'Clairac (47320)' => '47065', + 'Clairavaux (23500)' => '23063', + 'Claix (16440)' => '16101', + 'Clam (17500)' => '17108', + 'Claracq (64330)' => '64190', + 'Classun (40320)' => '40082', + 'Clavé (79420)' => '79092', + 'Clavette (17220)' => '17109', + 'Clèdes (40320)' => '40083', + 'Clérac (17270)' => '17110', + 'Clergoux (19320)' => '19056', + 'Clermont (40180)' => '40084', + 'Clermont-d\'Excideuil (24160)' => '24124', + 'Clermont-de-Beauregard (24140)' => '24123', + 'Clermont-Dessous (47130)' => '47066', + 'Clermont-Soubiran (47270)' => '47067', + 'Clessé (79350)' => '79094', + 'Cleyrac (33540)' => '33129', + 'Clion (17240)' => '17111', + 'Cloué (86600)' => '86080', + 'Clugnat (23270)' => '23064', + 'Clussais-la-Pommeraie (79190)' => '79095', + 'Coarraze (64800)' => '64191', + 'Cocumont (47250)' => '47068', + 'Cognac (16100)' => '16102', + 'Cognac-la-Forêt (87310)' => '87046', + 'Coimères (33210)' => '33130', + 'Coirac (33540)' => '33131', + 'Coivert (17330)' => '17114', + 'Colayrac-Saint-Cirq (47450)' => '47069', + 'Collonges-la-Rouge (19500)' => '19057', + 'Colombier (24560)' => '24126', + 'Colombiers (17460)' => '17115', + 'Colombiers (86490)' => '86081', + 'Colondannes (23800)' => '23065', + 'Coly (24120)' => '24127', + 'Comberanche-et-Épeluche (24600)' => '24128', + 'Combiers (16320)' => '16103', + 'Combrand (79140)' => '79096', + 'Combressol (19250)' => '19058', + 'Commensacq (40210)' => '40085', + 'Compreignac (87140)' => '87047', + 'Comps (33710)' => '33132', + 'Concèze (19350)' => '19059', + 'Conchez-de-Béarn (64330)' => '64192', + 'Condac (16700)' => '16104', + 'Condat-sur-Ganaveix (19140)' => '19060', + 'Condat-sur-Trincou (24530)' => '24129', + 'Condat-sur-Vézère (24570)' => '24130', + 'Condat-sur-Vienne (87920)' => '87048', + 'Condéon (16360)' => '16105', + 'Condezaygues (47500)' => '47070', + 'Confolens (16500)' => '16106', + 'Confolent-Port-Dieu (19200)' => '19167', + 'Conne-de-Labarde (24560)' => '24132', + 'Connezac (24300)' => '24131', + 'Consac (17150)' => '17116', + 'Contré (17470)' => '17117', + 'Corbère-Abères (64350)' => '64193', + 'Corgnac-sur-l\'Isle (24800)' => '24134', + 'Corignac (17130)' => '17118', + 'Corme-Écluse (17600)' => '17119', + 'Corme-Royal (17600)' => '17120', + 'Cornil (19150)' => '19061', + 'Cornille (24750)' => '24135', + 'Corrèze (19800)' => '19062', + 'Coslédaà-Lube-Boast (64160)' => '64194', + 'Cosnac (19360)' => '19063', + 'Coubeyrac (33890)' => '33133', + 'Coubjours (24390)' => '24136', + 'Coublucq (64410)' => '64195', + 'Coudures (40500)' => '40086', + 'Couffy-sur-Sarsonne (19340)' => '19064', + 'Couhé (86700)' => '86082', + 'Coulaures (24420)' => '24137', + 'Coulgens (16560)' => '16107', + 'Coulombiers (86600)' => '86083', + 'Coulon (79510)' => '79100', + 'Coulonges (16330)' => '16108', + 'Coulonges (17800)' => '17122', + 'Coulonges (86290)' => '86084', + 'Coulonges-sur-l\'Autize (79160)' => '79101', + 'Coulonges-Thouarsais (79330)' => '79102', + 'Coulounieix-Chamiers (24660)' => '24138', + 'Coulx (47260)' => '47071', + 'Couquèques (33340)' => '33134', + 'Courant (17330)' => '17124', + 'Courbiac (47370)' => '47072', + 'Courbillac (16200)' => '16109', + 'Courcelles (17400)' => '17125', + 'Courcerac (17160)' => '17126', + 'Courcôme (16240)' => '16110', + 'Courçon (17170)' => '17127', + 'Courcoury (17100)' => '17128', + 'Courgeac (16190)' => '16111', + 'Courlac (16210)' => '16112', + 'Courlay (79440)' => '79103', + 'Courpiac (33760)' => '33135', + 'Courpignac (17130)' => '17129', + 'Cours (47360)' => '47073', + 'Cours (79220)' => '79104', + 'Cours-de-Monségur (33580)' => '33136', + 'Cours-de-Pile (24520)' => '24140', + 'Cours-les-Bains (33690)' => '33137', + 'Coursac (24430)' => '24139', + 'Courteix (19340)' => '19065', + 'Coussac-Bonneval (87500)' => '87049', + 'Coussay (86110)' => '86085', + 'Coussay-les-Bois (86270)' => '86086', + 'Couthures-sur-Garonne (47180)' => '47074', + 'Coutières (79340)' => '79105', + 'Coutras (33230)' => '33138', + 'Couture (16460)' => '16114', + 'Couture-d\'Argenson (79110)' => '79106', + 'Coutures (24320)' => '24141', + 'Coutures (33580)' => '33139', + 'Coux (17130)' => '17130', + 'Coux et Bigaroque-Mouzens (24220)' => '24142', + 'Couze-et-Saint-Front (24150)' => '24143', + 'Couzeix (87270)' => '87050', + 'Cozes (17120)' => '17131', + 'Cramchaban (17170)' => '17132', + 'Craon (86110)' => '86087', + 'Cravans (17260)' => '17133', + 'Crazannes (17350)' => '17134', + 'Créon (33670)' => '33140', + 'Créon-d\'Armagnac (40240)' => '40087', + 'Cressac-Saint-Genis (16250)' => '16115', + 'Cressat (23140)' => '23068', + 'Cressé (17160)' => '17135', + 'Creyssac (24350)' => '24144', + 'Creysse (24100)' => '24145', + 'Creyssensac-et-Pissot (24380)' => '24146', + 'Crézières (79110)' => '79107', + 'Criteuil-la-Magdeleine (16300)' => '16116', + 'Crocq (23260)' => '23069', + 'Croignon (33750)' => '33141', + 'Croix-Chapeau (17220)' => '17136', + 'Cromac (87160)' => '87053', + 'Crouseilles (64350)' => '64196', + 'Croutelle (86240)' => '86088', + 'Crozant (23160)' => '23070', + 'Croze (23500)' => '23071', + 'Cubjac (24640)' => '24147', + 'Cublac (19520)' => '19066', + 'Cubnezais (33620)' => '33142', + 'Cubzac-les-Ponts (33240)' => '33143', + 'Cudos (33430)' => '33144', + 'Cuhon (86110)' => '86089', + 'Cunèges (24240)' => '24148', + 'Cuq (47220)' => '47076', + 'Cuqueron (64360)' => '64197', + 'Curac (16210)' => '16117', + 'Curçay-sur-Dive (86120)' => '86090', + 'Curemonte (19500)' => '19067', + 'Cursan (33670)' => '33145', + 'Curzay-sur-Vonne (86600)' => '86091', + 'Cussac (87150)' => '87054', + 'Cussac-Fort-Médoc (33460)' => '33146', + 'Cuzorn (47500)' => '47077', + 'Daglan (24250)' => '24150', + 'Daignac (33420)' => '33147', + 'Damazan (47160)' => '47078', + 'Dampierre-sur-Boutonne (17470)' => '17138', + 'Dampniat (19360)' => '19068', + 'Dangé-Saint-Romain (86220)' => '86092', + 'Darazac (19220)' => '19069', + 'Dardenac (33420)' => '33148', + 'Darnac (87320)' => '87055', + 'Darnets (19300)' => '19070', + 'Daubèze (33540)' => '33149', + 'Dausse (47140)' => '47079', + 'Davignac (19250)' => '19071', + 'Dax (40100)' => '40088', + 'Denguin (64230)' => '64198', + 'Dercé (86420)' => '86093', + 'Deviat (16190)' => '16118', + 'Dévillac (47210)' => '47080', + 'Dienné (86410)' => '86094', + 'Dieulivol (33580)' => '33150', + 'Dignac (16410)' => '16119', + 'Dinsac (87210)' => '87056', + 'Dirac (16410)' => '16120', + 'Dissay (86130)' => '86095', + 'Diusse (64330)' => '64199', + 'Doazit (40700)' => '40089', + 'Doazon (64370)' => '64200', + 'Doeuil-sur-le-Mignon (17330)' => '17139', + 'Dognen (64190)' => '64201', + 'Doissat (24170)' => '24151', + 'Dolmayrac (47110)' => '47081', + 'Dolus-d\'Oléron (17550)' => '17140', + 'Domeyrot (23140)' => '23072', + 'Domezain-Berraute (64120)' => '64202', + 'Domme (24250)' => '24152', + 'Dompierre-les-Églises (87190)' => '87057', + 'Dompierre-sur-Charente (17610)' => '17141', + 'Dompierre-sur-Mer (17139)' => '17142', + 'Domps (87120)' => '87058', + 'Dondas (47470)' => '47082', + 'Donnezac (33860)' => '33151', + 'Dontreix (23700)' => '23073', + 'Donzac (33410)' => '33152', + 'Donzacq (40360)' => '40090', + 'Donzenac (19270)' => '19072', + 'Douchapt (24350)' => '24154', + 'Doudrac (47210)' => '47083', + 'Doulezon (33350)' => '33153', + 'Doumy (64450)' => '64203', + 'Dournazac (87230)' => '87060', + 'Doussay (86140)' => '86096', + 'Douville (24140)' => '24155', + 'Doux (79390)' => '79108', + 'Douzains (47330)' => '47084', + 'Douzat (16290)' => '16121', + 'Douzillac (24190)' => '24157', + 'Droux (87190)' => '87061', + 'Duhort-Bachen (40800)' => '40091', + 'Dumes (40500)' => '40092', + 'Dun-le-Palestel (23800)' => '23075', + 'Durance (47420)' => '47085', + 'Duras (47120)' => '47086', + 'Dussac (24270)' => '24158', + 'Eaux-Bonnes (64440)' => '64204', + 'Ébréon (16140)' => '16122', + 'Échallat (16170)' => '16123', + 'Échebrune (17800)' => '17145', + 'Échillais (17620)' => '17146', + 'Échiré (79410)' => '79109', + 'Échourgnac (24410)' => '24159', + 'Écoyeux (17770)' => '17147', + 'Écuras (16220)' => '16124', + 'Écurat (17810)' => '17148', + 'Édon (16320)' => '16125', + 'Égletons (19300)' => '19073', + 'Église-Neuve-d\'Issac (24400)' => '24161', + 'Église-Neuve-de-Vergt (24380)' => '24160', + 'Empuré (16240)' => '16127', + 'Engayrac (47470)' => '47087', + 'Ensigné (79170)' => '79111', + 'Épannes (79270)' => '79112', + 'Épargnes (17120)' => '17152', + 'Épenède (16490)' => '16128', + 'Éraville (16120)' => '16129', + 'Escalans (40310)' => '40093', + 'Escassefort (47350)' => '47088', + 'Escaudes (33840)' => '33155', + 'Escaunets (65500)' => '65160', + 'Esclottes (47120)' => '47089', + 'Escoire (24420)' => '24162', + 'Escos (64270)' => '64205', + 'Escot (64490)' => '64206', + 'Escou (64870)' => '64207', + 'Escoubès (64160)' => '64208', + 'Escource (40210)' => '40094', + 'Escoussans (33760)' => '33156', + 'Escout (64870)' => '64209', + 'Escurès (64350)' => '64210', + 'Eslourenties-Daban (64420)' => '64211', + 'Esnandes (17137)' => '17153', + 'Espagnac (19150)' => '19075', + 'Espartignac (19140)' => '19076', + 'Espéchède (64160)' => '64212', + 'Espelette (64250)' => '64213', + 'Espès-Undurein (64130)' => '64214', + 'Espiens (47600)' => '47090', + 'Espiet (33420)' => '33157', + 'Espiute (64390)' => '64215', + 'Espoey (64420)' => '64216', + 'Esquiule (64400)' => '64217', + 'Esse (16500)' => '16131', + 'Essouvert (17400)' => '17277', + 'Estérençuby (64220)' => '64218', + 'Estialescq (64290)' => '64219', + 'Estibeaux (40290)' => '40095', + 'Estigarde (40240)' => '40096', + 'Estillac (47310)' => '47091', + 'Estivals (19600)' => '19077', + 'Estivaux (19410)' => '19078', + 'Estos (64400)' => '64220', + 'Étagnac (16150)' => '16132', + 'Étaules (17750)' => '17155', + 'Étauliers (33820)' => '33159', + 'Etcharry (64120)' => '64221', + 'Etchebar (64470)' => '64222', + 'Étouars (24360)' => '24163', + 'Étriac (16250)' => '16133', + 'Etsaut (64490)' => '64223', + 'Eugénie-les-Bains (40320)' => '40097', + 'Évaux-les-Bains (23110)' => '23076', + 'Excideuil (24160)' => '24164', + 'Exideuil (16150)' => '16134', + 'Exireuil (79400)' => '79114', + 'Exoudun (79800)' => '79115', + 'Expiremont (17130)' => '17156', + 'Eybouleuf (87400)' => '87062', + 'Eyburie (19140)' => '19079', + 'Eygurande (19340)' => '19080', + 'Eygurande-et-Gardedeuil (24700)' => '24165', + 'Eyjeaux (87220)' => '87063', + 'Eyliac (24330)' => '24166', + 'Eymet (24500)' => '24167', + 'Eymouthiers (16220)' => '16135', + 'Eymoutiers (87120)' => '87064', + 'Eynesse (33220)' => '33160', + 'Eyrans (33390)' => '33161', + 'Eyrein (19800)' => '19081', + 'Eyres-Moncube (40500)' => '40098', + 'Eysines (33320)' => '33162', + 'Eysus (64400)' => '64224', + 'Eyvirat (24460)' => '24170', + 'Eyzerac (24800)' => '24171', + 'Faleyras (33760)' => '33163', + 'Fals (47220)' => '47092', + 'Fanlac (24290)' => '24174', + 'Fargues (33210)' => '33164', + 'Fargues (40500)' => '40099', + 'Fargues-Saint-Hilaire (33370)' => '33165', + 'Fargues-sur-Ourbise (47700)' => '47093', + 'Fauguerolles (47400)' => '47094', + 'Fauillet (47400)' => '47095', + 'Faurilles (24560)' => '24176', + 'Faux (24560)' => '24177', + 'Faux-la-Montagne (23340)' => '23077', + 'Faux-Mazuras (23400)' => '23078', + 'Favars (19330)' => '19082', + 'Faye-l\'Abbesse (79350)' => '79116', + 'Faye-sur-Ardin (79160)' => '79117', + 'Féas (64570)' => '64225', + 'Felletin (23500)' => '23079', + 'Fénery (79450)' => '79118', + 'Féniers (23100)' => '23080', + 'Fenioux (17350)' => '17157', + 'Fenioux (79160)' => '79119', + 'Ferrensac (47330)' => '47096', + 'Ferrières (17170)' => '17158', + 'Festalemps (24410)' => '24178', + 'Feugarolles (47230)' => '47097', + 'Feuillade (16380)' => '16137', + 'Feyt (19340)' => '19083', + 'Feytiat (87220)' => '87065', + 'Fichous-Riumayou (64410)' => '64226', + 'Fieux (47600)' => '47098', + 'Firbeix (24450)' => '24180', + 'Flaugeac (24240)' => '24181', + 'Flaujagues (33350)' => '33168', + 'Flavignac (87230)' => '87066', + 'Flayat (23260)' => '23081', + 'Fléac (16730)' => '16138', + 'Fléac-sur-Seugne (17800)' => '17159', + 'Fleix (86300)' => '86098', + 'Fleurac (16200)' => '16139', + 'Fleurac (24580)' => '24183', + 'Fleurat (23320)' => '23082', + 'Fleuré (86340)' => '86099', + 'Floirac (17120)' => '17160', + 'Floirac (33270)' => '33167', + 'Florimont-Gaumier (24250)' => '24184', + 'Floudès (33190)' => '33169', + 'Folles (87250)' => '87067', + 'Fomperron (79340)' => '79121', + 'Fongrave (47260)' => '47099', + 'Fonroque (24500)' => '24186', + 'Fontaine-Chalendray (17510)' => '17162', + 'Fontaine-le-Comte (86240)' => '86100', + 'Fontaines-d\'Ozillac (17500)' => '17163', + 'Fontanières (23110)' => '23083', + 'Fontclaireau (16230)' => '16140', + 'Fontcouverte (17100)' => '17164', + 'Fontenet (17400)' => '17165', + 'Fontenille (16230)' => '16141', + 'Fontenille-Saint-Martin-d\'Entraigues (79110)' => '79122', + 'Fontet (33190)' => '33170', + 'Forges (17290)' => '17166', + 'Forgès (19380)' => '19084', + 'Fors (79230)' => '79125', + 'Fossemagne (24210)' => '24188', + 'Fossès-et-Baleyssac (33190)' => '33171', + 'Fougueyrolles (33220)' => '24189', + 'Foulayronnes (47510)' => '47100', + 'Fouleix (24380)' => '24190', + 'Fouquebrune (16410)' => '16143', + 'Fouqueure (16140)' => '16144', + 'Fouras (17450)' => '17168', + 'Fourques-sur-Garonne (47200)' => '47101', + 'Fours (33390)' => '33172', + 'Foussignac (16200)' => '16145', + 'Fraisse (24130)' => '24191', + 'Francescas (47600)' => '47102', + 'François (79260)' => '79128', + 'Francs (33570)' => '33173', + 'Fransèches (23480)' => '23086', + 'Fréchou (47600)' => '47103', + 'Frégimont (47360)' => '47104', + 'Frespech (47140)' => '47105', + 'Fresselines (23450)' => '23087', + 'Fressines (79370)' => '79129', + 'Fromental (87250)' => '87068', + 'Fronsac (33126)' => '33174', + 'Frontenac (33760)' => '33175', + 'Frontenay-Rohan-Rohan (79270)' => '79130', + 'Frozes (86190)' => '86102', + 'Fumel (47500)' => '47106', + 'Gaas (40350)' => '40101', + 'Gabarnac (33410)' => '33176', + 'Gabarret (40310)' => '40102', + 'Gabaston (64160)' => '64227', + 'Gabat (64120)' => '64228', + 'Gabillou (24210)' => '24192', + 'Gageac-et-Rouillac (24240)' => '24193', + 'Gaillan-en-Médoc (33340)' => '33177', + 'Gaillères (40090)' => '40103', + 'Gajac (33430)' => '33178', + 'Gajoubert (87330)' => '87069', + 'Galapian (47190)' => '47107', + 'Galgon (33133)' => '33179', + 'Gamarde-les-Bains (40380)' => '40104', + 'Gamarthe (64220)' => '64229', + 'Gan (64290)' => '64230', + 'Gans (33430)' => '33180', + 'Garat (16410)' => '16146', + 'Gardegan-et-Tourtirac (33350)' => '33181', + 'Gardères (65320)' => '65185', + 'Gardes-le-Pontaroux (16320)' => '16147', + 'Gardonne (24680)' => '24194', + 'Garein (40420)' => '40105', + 'Garindein (64130)' => '64231', + 'Garlède-Mondebat (64450)' => '64232', + 'Garlin (64330)' => '64233', + 'Garos (64410)' => '64234', + 'Garrey (40180)' => '40106', + 'Garris (64120)' => '64235', + 'Garrosse (40110)' => '40107', + 'Gartempe (23320)' => '23088', + 'Gastes (40160)' => '40108', + 'Gaugeac (24540)' => '24195', + 'Gaujac (47200)' => '47108', + 'Gaujacq (40330)' => '40109', + 'Gauriac (33710)' => '33182', + 'Gauriaguet (33240)' => '33183', + 'Gavaudun (47150)' => '47109', + 'Gayon (64350)' => '64236', + 'Geaune (40320)' => '40110', + 'Geay (17250)' => '17171', + 'Geay (79330)' => '79131', + 'Gelos (64110)' => '64237', + 'Geloux (40090)' => '40111', + 'Gémozac (17260)' => '17172', + 'Genac-Bignac (16170)' => '16148', + 'Gençay (86160)' => '86103', + 'Générac (33920)' => '33184', + 'Génis (24160)' => '24196', + 'Génissac (33420)' => '33185', + 'Genneton (79150)' => '79132', + 'Genouillac (16270)' => '16149', + 'Genouillac (23350)' => '23089', + 'Genouillé (17430)' => '17174', + 'Genouillé (86250)' => '86104', + 'Gensac (33890)' => '33186', + 'Gensac-la-Pallue (16130)' => '16150', + 'Genté (16130)' => '16151', + 'Gentioux-Pigerolles (23340)' => '23090', + 'Ger (64530)' => '64238', + 'Gerderest (64160)' => '64239', + 'Gère-Bélesten (64260)' => '64240', + 'Germignac (17520)' => '17175', + 'Germond-Rouvre (79220)' => '79133', + 'Géronce (64400)' => '64241', + 'Gestas (64190)' => '64242', + 'Géus-d\'Arzacq (64370)' => '64243', + 'Geüs-d\'Oloron (64400)' => '64244', + 'Gibourne (17160)' => '17176', + 'Gibret (40380)' => '40112', + 'Gimel-les-Cascades (19800)' => '19085', + 'Gimeux (16130)' => '16152', + 'Ginestet (24130)' => '24197', + 'Gioux (23500)' => '23091', + 'Gironde-sur-Dropt (33190)' => '33187', + 'Giscos (33840)' => '33188', + 'Givrezac (17260)' => '17178', + 'Gizay (86340)' => '86105', + 'Glandon (87500)' => '87071', + 'Glanges (87380)' => '87072', + 'Glénay (79330)' => '79134', + 'Glénic (23380)' => '23092', + 'Glénouze (86200)' => '86106', + 'Goès (64400)' => '64245', + 'Gomer (64420)' => '64246', + 'Gond-Pontouvre (16160)' => '16154', + 'Gondeville (16200)' => '16153', + 'Gontaud-de-Nogaret (47400)' => '47110', + 'Goos (40180)' => '40113', + 'Gornac (33540)' => '33189', + 'Gorre (87310)' => '87073', + 'Gotein-Libarrenx (64130)' => '64247', + 'Goualade (33840)' => '33190', + 'Gouex (86320)' => '86107', + 'Goulles (19430)' => '19086', + 'Gourbera (40990)' => '40114', + 'Gourdon-Murat (19170)' => '19087', + 'Gourgé (79200)' => '79135', + 'Gournay-Loizé (79110)' => '79136', + 'Gours (33660)' => '33191', + 'Gourville (16170)' => '16156', + 'Gourvillette (17490)' => '17180', + 'Gousse (40465)' => '40115', + 'Gout-Rossignol (24320)' => '24199', + 'Gouts (40400)' => '40116', + 'Gouzon (23230)' => '23093', + 'Gradignan (33170)' => '33192', + 'Grand-Brassac (24350)' => '24200', + 'Grandjean (17350)' => '17181', + 'Grandsaigne (19300)' => '19088', + 'Granges-d\'Ans (24390)' => '24202', + 'Granges-sur-Lot (47260)' => '47111', + 'Granzay-Gript (79360)' => '79137', + 'Grassac (16380)' => '16158', + 'Grateloup-Saint-Gayrand (47400)' => '47112', + 'Graves-Saint-Amant (16120)' => '16297', + 'Grayan-et-l\'Hôpital (33590)' => '33193', + 'Grayssas (47270)' => '47113', + 'Grenade-sur-l\'Adour (40270)' => '40117', + 'Grézac (17120)' => '17183', + 'Grèzes (24120)' => '24204', + 'Grézet-Cavagnan (47250)' => '47114', + 'Grézillac (33420)' => '33194', + 'Grignols (24110)' => '24205', + 'Grignols (33690)' => '33195', + 'Grives (24170)' => '24206', + 'Groléjac (24250)' => '24207', + 'Gros-Chastang (19320)' => '19089', + 'Grun-Bordas (24380)' => '24208', + 'Guéret (23000)' => '23096', + 'Guérin (47250)' => '47115', + 'Guesnes (86420)' => '86109', + 'Guéthary (64210)' => '64249', + 'Guiche (64520)' => '64250', + 'Guillac (33420)' => '33196', + 'Guillos (33720)' => '33197', + 'Guimps (16300)' => '16160', + 'Guinarthe-Parenties (64390)' => '64251', + 'Guitinières (17500)' => '17187', + 'Guîtres (33230)' => '33198', + 'Guizengeard (16480)' => '16161', + 'Gujan-Mestras (33470)' => '33199', + 'Gumond (19320)' => '19090', + 'Gurat (16320)' => '16162', + 'Gurmençon (64400)' => '64252', + 'Gurs (64190)' => '64253', + 'Habas (40290)' => '40118', + 'Hagetaubin (64370)' => '64254', + 'Hagetmau (40700)' => '40119', + 'Haimps (17160)' => '17188', + 'Haims (86310)' => '86110', + 'Halsou (64480)' => '64255', + 'Hanc (79110)' => '79140', + 'Hasparren (64240)' => '64256', + 'Hastingues (40300)' => '40120', + 'Hauriet (40250)' => '40121', + 'Haut-de-Bosdarros (64800)' => '64257', + 'Haut-Mauco (40280)' => '40122', + 'Hautefage (19400)' => '19091', + 'Hautefage-la-Tour (47340)' => '47117', + 'Hautefaye (24300)' => '24209', + 'Hautefort (24390)' => '24210', + 'Hautesvignes (47400)' => '47118', + 'Haux (33550)' => '33201', + 'Haux (64470)' => '64258', + 'Hélette (64640)' => '64259', + 'Hendaye (64700)' => '64260', + 'Herm (40990)' => '40123', + 'Herré (40310)' => '40124', + 'Herrère (64680)' => '64261', + 'Heugas (40180)' => '40125', + 'Hiers-Brouage (17320)' => '17189', + 'Hiersac (16290)' => '16163', + 'Hiesse (16490)' => '16164', + 'Higuères-Souye (64160)' => '64262', + 'Hinx (40180)' => '40126', + 'Hontanx (40190)' => '40127', + 'Horsarrieu (40700)' => '40128', + 'Hosta (64120)' => '64265', + 'Hostens (33125)' => '33202', + 'Houeillès (47420)' => '47119', + 'Houlette (16200)' => '16165', + 'Hours (64420)' => '64266', + 'Hourtin (33990)' => '33203', + 'Hure (33190)' => '33204', + 'Ibarrolle (64120)' => '64267', + 'Idaux-Mendy (64130)' => '64268', + 'Idron (64320)' => '64269', + 'Igon (64800)' => '64270', + 'Iholdy (64640)' => '64271', + 'Île-d\'Aix (17123)' => '17004', + 'Ilharre (64120)' => '64272', + 'Illats (33720)' => '33205', + 'Ingrandes (86220)' => '86111', + 'Irais (79600)' => '79141', + 'Irissarry (64780)' => '64273', + 'Irouléguy (64220)' => '64274', + 'Isle (87170)' => '87075', + 'Isle-Saint-Georges (33640)' => '33206', + 'Ispoure (64220)' => '64275', + 'Issac (24400)' => '24211', + 'Issigeac (24560)' => '24212', + 'Issor (64570)' => '64276', + 'Issoudun-Létrieix (23130)' => '23097', + 'Isturits (64240)' => '64277', + 'Iteuil (86240)' => '86113', + 'Itxassou (64250)' => '64279', + 'Izeste (64260)' => '64280', + 'Izon (33450)' => '33207', + 'Jabreilles-les-Bordes (87370)' => '87076', + 'Jalesches (23270)' => '23098', + 'Janailhac (87800)' => '87077', + 'Janaillat (23250)' => '23099', + 'Jardres (86800)' => '86114', + 'Jarnac (16200)' => '16167', + 'Jarnac-Champagne (17520)' => '17192', + 'Jarnages (23140)' => '23100', + 'Jasses (64190)' => '64281', + 'Jatxou (64480)' => '64282', + 'Jau-Dignac-et-Loirac (33590)' => '33208', + 'Jauldes (16560)' => '16168', + 'Jaunay-Clan (86130)' => '86115', + 'Jaure (24140)' => '24213', + 'Javerdat (87520)' => '87078', + 'Javerlhac-et-la-Chapelle-Saint-Robert (24300)' => '24214', + 'Javrezac (16100)' => '16169', + 'Jaxu (64220)' => '64283', + 'Jayac (24590)' => '24215', + 'Jazeneuil (86600)' => '86116', + 'Jazennes (17260)' => '17196', + 'Jonzac (17500)' => '17197', + 'Josse (40230)' => '40129', + 'Jouac (87890)' => '87080', + 'Jouhet (86500)' => '86117', + 'Jouillat (23220)' => '23101', + 'Jourgnac (87800)' => '87081', + 'Journet (86290)' => '86118', + 'Journiac (24260)' => '24217', + 'Joussé (86350)' => '86119', + 'Jugazan (33420)' => '33209', + 'Jugeals-Nazareth (19500)' => '19093', + 'Juicq (17770)' => '17198', + 'Juignac (16190)' => '16170', + 'Juillac (19350)' => '19094', + 'Juillac (33890)' => '33210', + 'Juillac-le-Coq (16130)' => '16171', + 'Juillé (16230)' => '16173', + 'Juillé (79170)' => '79142', + 'Julienne (16200)' => '16174', + 'Jumilhac-le-Grand (24630)' => '24218', + 'Jurançon (64110)' => '64284', + 'Juscorps (79230)' => '79144', + 'Jusix (47180)' => '47120', + 'Jussas (17130)' => '17199', + 'Juxue (64120)' => '64285', + 'L\'Absie (79240)' => '79001', + 'L\'Église-aux-Bois (19170)' => '19074', + 'L\'Éguille (17600)' => '17151', + 'L\'Hôpital-d\'Orion (64270)' => '64263', + 'L\'Hôpital-Saint-Blaise (64130)' => '64264', + 'L\'Houmeau (17137)' => '17190', + 'L\'Isle-d\'Espagnac (16340)' => '16166', + 'L\'Isle-Jourdain (86150)' => '86112', + 'La Bachellerie (24210)' => '24020', + 'La Barde (17360)' => '17033', + 'La Bastide-Clairence (64240)' => '64289', + 'La Bataille (79110)' => '79027', + 'La Bazeuge (87210)' => '87008', + 'La Boissière-d\'Ans (24640)' => '24047', + 'La Boissière-en-Gâtine (79310)' => '79040', + 'La Brède (33650)' => '33213', + 'La Brée-les-Bains (17840)' => '17486', + 'La Brionne (23000)' => '23033', + 'La Brousse (17160)' => '17071', + 'La Bussière (86310)' => '86040', + 'La Cassagne (24120)' => '24085', + 'La Celle-Dunoise (23800)' => '23039', + 'La Celle-sous-Gouzon (23230)' => '23040', + 'La Cellette (23350)' => '23041', + 'La Chapelle (16140)' => '16081', + 'La Chapelle-Aubareil (24290)' => '24106', + 'La Chapelle-aux-Brocs (19360)' => '19043', + 'La Chapelle-aux-Saints (19120)' => '19044', + 'La Chapelle-Baloue (23160)' => '23050', + 'La Chapelle-Bâton (79220)' => '79070', + 'La Chapelle-Bâton (86250)' => '86055', + 'La Chapelle-Bertrand (79200)' => '79071', + 'La Chapelle-des-Pots (17100)' => '17089', + 'La Chapelle-Faucher (24530)' => '24107', + 'La Chapelle-Gonaguet (24350)' => '24108', + 'La Chapelle-Grésignac (24320)' => '24109', + 'La Chapelle-Montabourlet (24320)' => '24110', + 'La Chapelle-Montbrandeix (87440)' => '87037', + 'La Chapelle-Montmoreau (24300)' => '24111', + 'La Chapelle-Montreuil (86470)' => '86056', + 'La Chapelle-Moulière (86210)' => '86058', + 'La Chapelle-Pouilloux (79190)' => '79074', + 'La Chapelle-Saint-Étienne (79240)' => '79075', + 'La Chapelle-Saint-Géraud (19430)' => '19045', + 'La Chapelle-Saint-Jean (24390)' => '24113', + 'La Chapelle-Saint-Laurent (79430)' => '79076', + 'La Chapelle-Saint-Martial (23250)' => '23051', + 'La Chapelle-Taillefert (23000)' => '23052', + 'La Chapelle-Thireuil (79160)' => '79077', + 'La Chaussade (23200)' => '23059', + 'La Chaussée (86330)' => '86069', + 'La Chèvrerie (16240)' => '16098', + 'La Clisse (17600)' => '17112', + 'La Clotte (17360)' => '17113', + 'La Coquille (24450)' => '24133', + 'La Couarde (79800)' => '79098', + 'La Couarde-sur-Mer (17670)' => '17121', + 'La Couronne (16400)' => '16113', + 'La Courtine (23100)' => '23067', + 'La Crèche (79260)' => '79048', + 'La Croisille-sur-Briance (87130)' => '87051', + 'La Croix-Blanche (47340)' => '47075', + 'La Croix-Comtesse (17330)' => '17137', + 'La Croix-sur-Gartempe (87210)' => '87052', + 'La Dornac (24120)' => '24153', + 'La Douze (24330)' => '24156', + 'La Faye (16700)' => '16136', + 'La Ferrière-Airoux (86160)' => '86097', + 'La Ferrière-en-Parthenay (79390)' => '79120', + 'La Feuillade (24120)' => '24179', + 'La Flotte (17630)' => '17161', + 'La Force (24130)' => '24222', + 'La Forêt-de-Tessé (16240)' => '16142', + 'La Forêt-du-Temple (23360)' => '23084', + 'La Forêt-sur-Sèvre (79380)' => '79123', + 'La Foye-Monjault (79360)' => '79127', + 'La Frédière (17770)' => '17169', + 'La Genétouze (17360)' => '17173', + 'La Geneytouse (87400)' => '87070', + 'La Gonterie-Boulouneix (24310)' => '24198', + 'La Grève-sur-Mignon (17170)' => '17182', + 'La Grimaudière (86330)' => '86108', + 'La Gripperie-Saint-Symphorien (17620)' => '17184', + 'La Jard (17460)' => '17191', + 'La Jarne (17220)' => '17193', + 'La Jarrie (17220)' => '17194', + 'La Jarrie-Audouin (17330)' => '17195', + 'La Jemaye (24410)' => '24216', + 'La Jonchère-Saint-Maurice (87340)' => '87079', + 'La Laigne (17170)' => '17201', + 'La Lande-de-Fronsac (33240)' => '33219', + 'La Magdeleine (16240)' => '16197', + 'La Mazière-aux-Bons-Hommes (23260)' => '23129', + 'La Meyze (87800)' => '87096', + 'La Mothe-Saint-Héray (79800)' => '79184', + 'La Nouaille (23500)' => '23144', + 'La Péruse (16270)' => '16259', + 'La Petite-Boissière (79700)' => '79207', + 'La Peyratte (79200)' => '79208', + 'La Porcherie (87380)' => '87120', + 'La Pouge (23250)' => '23157', + 'La Puye (86260)' => '86202', + 'La Réole (33190)' => '33352', + 'La Réunion (47700)' => '47222', + 'La Rivière (33126)' => '33356', + 'La Roche-Canillac (19320)' => '19174', + 'La Roche-Chalais (24490)' => '24354', + 'La Roche-l\'Abeille (87800)' => '87127', + 'La Roche-Posay (86270)' => '86207', + 'La Roche-Rigault (86200)' => '86079', + 'La Rochebeaucourt-et-Argentine (24340)' => '24353', + 'La Rochefoucauld (16110)' => '16281', + 'La Rochelle (17000)' => '17300', + 'La Rochénard (79270)' => '79229', + 'La Rochette (16110)' => '16282', + 'La Ronde (17170)' => '17303', + 'La Roque-Gageac (24250)' => '24355', + 'La Roquille (33220)' => '33360', + 'La Saunière (23000)' => '23169', + 'La Sauve (33670)' => '33505', + 'La Sauvetat-de-Savères (47270)' => '47289', + 'La Sauvetat-du-Dropt (47800)' => '47290', + 'La Sauvetat-sur-Lède (47150)' => '47291', + 'La Serre-Bussière-Vieille (23190)' => '23172', + 'La Souterraine (23300)' => '23176', + 'La Tâche (16260)' => '16377', + 'La Teste-de-Buch (33260)' => '33529', + 'La Tour-Blanche (24320)' => '24554', + 'La Tremblade (17390)' => '17452', + 'La Trimouille (86290)' => '86273', + 'La Vallée (17250)' => '17455', + 'La Vergne (17400)' => '17465', + 'La Villedieu (17470)' => '17471', + 'La Villedieu (23340)' => '23264', + 'La Villedieu-du-Clain (86340)' => '86290', + 'La Villeneuve (23260)' => '23265', + 'La Villetelle (23260)' => '23266', + 'Laà-Mondrans (64300)' => '64286', + 'Laàs (64390)' => '64287', + 'Labarde (33460)' => '33211', + 'Labastide-Castel-Amouroux (47250)' => '47121', + 'Labastide-Cézéracq (64170)' => '64288', + 'Labastide-Chalosse (40700)' => '40130', + 'Labastide-d\'Armagnac (40240)' => '40131', + 'Labastide-Monréjeau (64170)' => '64290', + 'Labastide-Villefranche (64270)' => '64291', + 'Labatmale (64530)' => '64292', + 'Labatut (40300)' => '40132', + 'Labatut (64460)' => '64293', + 'Labenne (40530)' => '40133', + 'Labescau (33690)' => '33212', + 'Labets-Biscay (64120)' => '64294', + 'Labeyrie (64300)' => '64295', + 'Labouheyre (40210)' => '40134', + 'Labretonie (47350)' => '47122', + 'Labrit (40420)' => '40135', + 'Lacadée (64300)' => '64296', + 'Lacajunte (40320)' => '40136', + 'Lacanau (33680)' => '33214', + 'Lacapelle-Biron (47150)' => '47123', + 'Lacarre (64220)' => '64297', + 'Lacarry-Arhan-Charritte-de-Haut (64470)' => '64298', + 'Lacaussade (47150)' => '47124', + 'Lacelle (19170)' => '19095', + 'Lacépède (47360)' => '47125', + 'Lachaise (16300)' => '16176', + 'Lachapelle (47350)' => '47126', + 'Lacommande (64360)' => '64299', + 'Lacq (64170)' => '64300', + 'Lacquy (40120)' => '40137', + 'Lacrabe (40700)' => '40138', + 'Lacropte (24380)' => '24220', + 'Ladapeyre (23270)' => '23102', + 'Ladaux (33760)' => '33215', + 'Ladignac-le-Long (87500)' => '87082', + 'Ladignac-sur-Rondelles (19150)' => '19096', + 'Ladiville (16120)' => '16177', + 'Lados (33124)' => '33216', + 'Lafage-sur-Sombre (19320)' => '19097', + 'Lafat (23800)' => '23103', + 'Lafitte-sur-Lot (47320)' => '47127', + 'Lafox (47240)' => '47128', + 'Lagarde-Enval (19150)' => '19098', + 'Lagarde-sur-le-Né (16300)' => '16178', + 'Lagarrigue (47190)' => '47129', + 'Lageon (79200)' => '79145', + 'Lagleygeolle (19500)' => '19099', + 'Laglorieuse (40090)' => '40139', + 'Lagor (64150)' => '64301', + 'Lagorce (33230)' => '33218', + 'Lagord (17140)' => '17200', + 'Lagos (64800)' => '64302', + 'Lagrange (40240)' => '40140', + 'Lagraulière (19700)' => '19100', + 'Lagruère (47400)' => '47130', + 'Laguenne (19150)' => '19101', + 'Laguinge-Restoue (64470)' => '64303', + 'Lagupie (47180)' => '47131', + 'Lahonce (64990)' => '64304', + 'Lahontan (64270)' => '64305', + 'Lahosse (40250)' => '40141', + 'Lahourcade (64150)' => '64306', + 'Lalande-de-Pomerol (33500)' => '33222', + 'Lalandusse (47330)' => '47132', + 'Lalinde (24150)' => '24223', + 'Lalongue (64350)' => '64307', + 'Lalonquette (64450)' => '64308', + 'Laluque (40465)' => '40142', + 'Lamarque (33460)' => '33220', + 'Lamayou (64460)' => '64309', + 'Lamazière-Basse (19160)' => '19102', + 'Lamazière-Haute (19340)' => '19103', + 'Lamongerie (19510)' => '19104', + 'Lamontjoie (47310)' => '47133', + 'Lamonzie-Montastruc (24520)' => '24224', + 'Lamonzie-Saint-Martin (24680)' => '24225', + 'Lamothe (40250)' => '40143', + 'Lamothe-Landerron (33190)' => '33221', + 'Lamothe-Montravel (24230)' => '24226', + 'Landerrouat (33790)' => '33223', + 'Landerrouet-sur-Ségur (33540)' => '33224', + 'Landes (17380)' => '17202', + 'Landiras (33720)' => '33225', + 'Landrais (17290)' => '17203', + 'Langoiran (33550)' => '33226', + 'Langon (33210)' => '33227', + 'Lanne-en-Barétous (64570)' => '64310', + 'Lannecaube (64350)' => '64311', + 'Lanneplaà (64300)' => '64312', + 'Lannes (47170)' => '47134', + 'Lanouaille (24270)' => '24227', + 'Lanquais (24150)' => '24228', + 'Lansac (33710)' => '33228', + 'Lantabat (64640)' => '64313', + 'Lanteuil (19190)' => '19105', + 'Lanton (33138)' => '33229', + 'Laparade (47260)' => '47135', + 'Laperche (47800)' => '47136', + 'Lapleau (19550)' => '19106', + 'Laplume (47310)' => '47137', + 'Lapouyade (33620)' => '33230', + 'Laprade (16390)' => '16180', + 'Larbey (40250)' => '40144', + 'Larceveau-Arros-Cibits (64120)' => '64314', + 'Larche (19600)' => '19107', + 'Largeasse (79240)' => '79147', + 'Laroche-près-Feyt (19340)' => '19108', + 'Laroin (64110)' => '64315', + 'Laroque (33410)' => '33231', + 'Laroque-Timbaut (47340)' => '47138', + 'Larrau (64560)' => '64316', + 'Larressore (64480)' => '64317', + 'Larreule (64410)' => '64318', + 'Larribar-Sorhapuru (64120)' => '64319', + 'Larrivière-Saint-Savin (40270)' => '40145', + 'Lartigue (33840)' => '33232', + 'Laruns (64440)' => '64320', + 'Laruscade (33620)' => '33233', + 'Larzac (24170)' => '24230', + 'Lascaux (19130)' => '19109', + 'Lasclaveries (64450)' => '64321', + 'Lasse (64220)' => '64322', + 'Lasserre (47600)' => '47139', + 'Lasserre (64350)' => '64323', + 'Lasseube (64290)' => '64324', + 'Lasseubetat (64290)' => '64325', + 'Lathus-Saint-Rémy (86390)' => '86120', + 'Latillé (86190)' => '86121', + 'Latresne (33360)' => '33234', + 'Latrille (40800)' => '40146', + 'Latronche (19160)' => '19110', + 'Laugnac (47360)' => '47140', + 'Laurède (40250)' => '40147', + 'Lauret (40320)' => '40148', + 'Laurière (87370)' => '87083', + 'Laussou (47150)' => '47141', + 'Lauthiers (86300)' => '86122', + 'Lauzun (47410)' => '47142', + 'Laval-sur-Luzège (19550)' => '19111', + 'Lavalade (24540)' => '24231', + 'Lavardac (47230)' => '47143', + 'Lavaufranche (23600)' => '23104', + 'Lavaur (24550)' => '24232', + 'Lavausseau (86470)' => '86123', + 'Lavaveix-les-Mines (23150)' => '23105', + 'Lavazan (33690)' => '33235', + 'Lavergne (47800)' => '47144', + 'Laveyssière (24130)' => '24233', + 'Lavignac (87230)' => '87084', + 'Lavoux (86800)' => '86124', + 'Lay-Lamidou (64190)' => '64326', + 'Layrac (47390)' => '47145', + 'Le Barp (33114)' => '33029', + 'Le Beugnon (79130)' => '79035', + 'Le Bois-Plage-en-Ré (17580)' => '17051', + 'Le Bouchage (16350)' => '16054', + 'Le Bourdeix (24300)' => '24056', + 'Le Bourdet (79210)' => '79046', + 'Le Bourg-d\'Hem (23220)' => '23029', + 'Le Bouscat (33110)' => '33069', + 'Le Breuil-Bernard (79320)' => '79051', + 'Le Bugue (24260)' => '24067', + 'Le Buis (87140)' => '87023', + 'Le Buisson-de-Cadouin (24480)' => '24068', + 'Le Busseau (79240)' => '79059', + 'Le Chalard (87500)' => '87031', + 'Le Change (24640)' => '24103', + 'Le Chastang (19190)' => '19048', + 'Le Château-d\'Oléron (17480)' => '17093', + 'Le Châtenet-en-Dognon (87400)' => '87042', + 'Le Chauchet (23130)' => '23058', + 'Le Chay (17600)' => '17097', + 'Le Chillou (79600)' => '79089', + 'Le Compas (23700)' => '23066', + 'Le Donzeil (23480)' => '23074', + 'Le Dorat (87210)' => '87059', + 'Le Douhet (17100)' => '17143', + 'Le Fieu (33230)' => '33166', + 'Le Fleix (24130)' => '24182', + 'Le Fouilloux (17270)' => '17167', + 'Le Frêche (40190)' => '40100', + 'Le Gicq (17160)' => '17177', + 'Le Grand-Bourg (23240)' => '23095', + 'Le Grand-Madieu (16450)' => '16157', + 'Le Grand-Village-Plage (17370)' => '17485', + 'Le Gua (17600)' => '17185', + 'Le Gué-d\'Alleré (17540)' => '17186', + 'Le Haillan (33185)' => '33200', + 'Le Jardin (19300)' => '19092', + 'Le Lardin-Saint-Lazare (24570)' => '24229', + 'Le Leuy (40250)' => '40153', + 'Le Lindois (16310)' => '16188', + 'Le Lonzac (19470)' => '19118', + 'Le Mas-d\'Agenais (47430)' => '47159', + 'Le Mas-d\'Artige (23100)' => '23125', + 'Le Monteil-au-Vicomte (23460)' => '23134', + 'Le Mung (17350)' => '17252', + 'Le Nizan (33430)' => '33305', + 'Le Palais-sur-Vienne (87410)' => '87113', + 'Le Passage (47520)' => '47201', + 'Le Pescher (19190)' => '19163', + 'Le Pian-Médoc (33290)' => '33322', + 'Le Pian-sur-Garonne (33490)' => '33323', + 'Le Pin (17210)' => '17276', + 'Le Pin (79140)' => '79210', + 'Le Pizou (24700)' => '24329', + 'Le Porge (33680)' => '33333', + 'Le Pout (33670)' => '33335', + 'Le Puy (33580)' => '33345', + 'Le Retail (79130)' => '79226', + 'Le Rochereau (86170)' => '86208', + 'Le Sen (40420)' => '40297', + 'Le Seure (17770)' => '17426', + 'Le Taillan-Médoc (33320)' => '33519', + 'Le Tallud (79200)' => '79322', + 'Le Tâtre (16360)' => '16380', + 'Le Teich (33470)' => '33527', + 'Le Temple (33680)' => '33528', + 'Le Temple-sur-Lot (47110)' => '47306', + 'Le Thou (17290)' => '17447', + 'Le Tourne (33550)' => '33534', + 'Le Tuzan (33125)' => '33536', + 'Le Vanneau-Irleau (79270)' => '79337', + 'Le Verdon-sur-Mer (33123)' => '33544', + 'Le Vert (79170)' => '79346', + 'Le Vieux-Cérier (16350)' => '16403', + 'Le Vigeant (86150)' => '86289', + 'Le Vigen (87110)' => '87205', + 'Le Vignau (40270)' => '40329', + 'Lecumberry (64220)' => '64327', + 'Lédat (47300)' => '47146', + 'Ledeuix (64400)' => '64328', + 'Lée (64320)' => '64329', + 'Lées-Athas (64490)' => '64330', + 'Lège-Cap-Ferret (33950)' => '33236', + 'Léguillac-de-Cercles (24340)' => '24235', + 'Léguillac-de-l\'Auche (24110)' => '24236', + 'Leigné-les-Bois (86450)' => '86125', + 'Leigné-sur-Usseau (86230)' => '86127', + 'Leignes-sur-Fontaine (86300)' => '86126', + 'Lembeye (64350)' => '64331', + 'Lembras (24100)' => '24237', + 'Lème (64450)' => '64332', + 'Lempzours (24800)' => '24238', + 'Lencloître (86140)' => '86128', + 'Lencouacq (40120)' => '40149', + 'Léogeats (33210)' => '33237', + 'Léognan (33850)' => '33238', + 'Léon (40550)' => '40150', + 'Léoville (17500)' => '17204', + 'Lépaud (23170)' => '23106', + 'Lépinas (23150)' => '23107', + 'Léren (64270)' => '64334', + 'Lerm-et-Musset (33840)' => '33239', + 'Les Adjots (16700)' => '16002', + 'Les Alleuds (79190)' => '79006', + 'Les Angles-sur-Corrèze (19000)' => '19009', + 'Les Artigues-de-Lussac (33570)' => '33014', + 'Les Billanges (87340)' => '87016', + 'Les Billaux (33500)' => '33052', + 'Les Cars (87230)' => '87029', + 'Les Éduts (17510)' => '17149', + 'Les Églises-d\'Argenteuil (17400)' => '17150', + 'Les Églisottes-et-Chalaures (33230)' => '33154', + 'Les Essards (16210)' => '16130', + 'Les Essards (17250)' => '17154', + 'Les Esseintes (33190)' => '33158', + 'Les Eyzies-de-Tayac-Sireuil (24620)' => '24172', + 'Les Farges (24290)' => '24175', + 'Les Forges (79340)' => '79124', + 'Les Fosses (79360)' => '79126', + 'Les Gonds (17100)' => '17179', + 'Les Gours (16140)' => '16155', + 'Les Grands-Chézeaux (87160)' => '87074', + 'Les Graulges (24340)' => '24203', + 'Les Groseillers (79220)' => '79139', + 'Les Lèches (24400)' => '24234', + 'Les Lèves-et-Thoumeyragues (33220)' => '33242', + 'Les Mars (23700)' => '23123', + 'Les Mathes (17570)' => '17225', + 'Les Métairies (16200)' => '16220', + 'Les Nouillers (17380)' => '17266', + 'Les Ormes (86220)' => '86183', + 'Les Peintures (33230)' => '33315', + 'Les Pins (16260)' => '16261', + 'Les Portes-en-Ré (17880)' => '17286', + 'Les Salles-de-Castillon (33350)' => '33499', + 'Les Salles-Lavauguyon (87440)' => '87189', + 'Les Touches-de-Périgny (17160)' => '17451', + 'Les Trois-Moutiers (86120)' => '86274', + 'Lescar (64230)' => '64335', + 'Lescun (64490)' => '64336', + 'Lesgor (40400)' => '40151', + 'Lésignac-Durand (16310)' => '16183', + 'Lésigny (86270)' => '86129', + 'Lesparre-Médoc (33340)' => '33240', + 'Lesperon (40260)' => '40152', + 'Lespielle (64350)' => '64337', + 'Lespourcy (64160)' => '64338', + 'Lessac (16500)' => '16181', + 'Lestards (19170)' => '19112', + 'Lestelle-Bétharram (64800)' => '64339', + 'Lesterps (16420)' => '16182', + 'Lestiac-sur-Garonne (33550)' => '33241', + 'Leugny (86220)' => '86130', + 'Lévignac-de-Guyenne (47120)' => '47147', + 'Lévignacq (40170)' => '40154', + 'Leyrat (23600)' => '23108', + 'Leyritz-Moncassin (47700)' => '47148', + 'Lezay (79120)' => '79148', + 'Lhommaizé (86410)' => '86131', + 'Lhoumois (79390)' => '79149', + 'Libourne (33500)' => '33243', + 'Lichans-Sunhar (64470)' => '64340', + 'Lichères (16460)' => '16184', + 'Lichos (64130)' => '64341', + 'Licq-Athérey (64560)' => '64342', + 'Liginiac (19160)' => '19113', + 'Liglet (86290)' => '86132', + 'Lignan-de-Bazas (33430)' => '33244', + 'Lignan-de-Bordeaux (33360)' => '33245', + 'Lignareix (19200)' => '19114', + 'Ligné (16140)' => '16185', + 'Ligneyrac (19500)' => '19115', + 'Lignières-Sonneville (16130)' => '16186', + 'Ligueux (33220)' => '33246', + 'Ligugé (86240)' => '86133', + 'Limalonges (79190)' => '79150', + 'Limendous (64420)' => '64343', + 'Limeuil (24510)' => '24240', + 'Limeyrat (24210)' => '24241', + 'Limoges (87000)' => '87085', + 'Linard (23220)' => '23109', + 'Linards (87130)' => '87086', + 'Linars (16730)' => '16187', + 'Linazay (86400)' => '86134', + 'Liniers (86800)' => '86135', + 'Linxe (40260)' => '40155', + 'Liorac-sur-Louyre (24520)' => '24242', + 'Liourdres (19120)' => '19116', + 'Lioux-les-Monges (23700)' => '23110', + 'Liposthey (40410)' => '40156', + 'Lisle (24350)' => '24243', + 'Lissac-sur-Couze (19600)' => '19117', + 'Listrac-de-Durèze (33790)' => '33247', + 'Listrac-Médoc (33480)' => '33248', + 'Lit-et-Mixe (40170)' => '40157', + 'Livron (64530)' => '64344', + 'Lizant (86400)' => '86136', + 'Lizières (23240)' => '23111', + 'Lohitzun-Oyhercq (64120)' => '64345', + 'Loire-les-Marais (17870)' => '17205', + 'Loiré-sur-Nie (17470)' => '17206', + 'Loix (17111)' => '17207', + 'Lolme (24540)' => '24244', + 'Lombia (64160)' => '64346', + 'Lonçon (64410)' => '64347', + 'Londigny (16700)' => '16189', + 'Longèves (17230)' => '17208', + 'Longré (16240)' => '16190', + 'Longueville (47200)' => '47150', + 'Lonnes (16230)' => '16191', + 'Lons (64140)' => '64348', + 'Lonzac (17520)' => '17209', + 'Lorignac (17240)' => '17210', + 'Lorigné (79190)' => '79152', + 'Lormont (33310)' => '33249', + 'Losse (40240)' => '40158', + 'Lostanges (19500)' => '19119', + 'Loubejac (24550)' => '24245', + 'Loubens (33190)' => '33250', + 'Loubès-Bernac (47120)' => '47151', + 'Loubieng (64300)' => '64349', + 'Loubigné (79110)' => '79153', + 'Loubillé (79110)' => '79154', + 'Louchats (33125)' => '33251', + 'Loudun (86200)' => '86137', + 'Louer (40380)' => '40159', + 'Lougratte (47290)' => '47152', + 'Louhossoa (64250)' => '64350', + 'Louignac (19310)' => '19120', + 'Louin (79600)' => '79156', + 'Loulay (17330)' => '17211', + 'Loupes (33370)' => '33252', + 'Loupiac (33410)' => '33253', + 'Loupiac-de-la-Réole (33190)' => '33254', + 'Lourdios-Ichère (64570)' => '64351', + 'Lourdoueix-Saint-Pierre (23360)' => '23112', + 'Lourenties (64420)' => '64352', + 'Lourquen (40250)' => '40160', + 'Louvie-Juzon (64260)' => '64353', + 'Louvie-Soubiron (64440)' => '64354', + 'Louvigny (64410)' => '64355', + 'Louzac-Saint-André (16100)' => '16193', + 'Louzignac (17160)' => '17212', + 'Louzy (79100)' => '79157', + 'Lozay (17330)' => '17213', + 'Lubbon (40240)' => '40161', + 'Lubersac (19210)' => '19121', + 'Luc-Armau (64350)' => '64356', + 'Lucarré (64350)' => '64357', + 'Lucbardez-et-Bargues (40090)' => '40162', + 'Lucgarier (64420)' => '64358', + 'Luchapt (86430)' => '86138', + 'Luchat (17600)' => '17214', + 'Luché-sur-Brioux (79170)' => '79158', + 'Luché-Thouarsais (79330)' => '79159', + 'Lucmau (33840)' => '33255', + 'Lucq-de-Béarn (64360)' => '64359', + 'Ludon-Médoc (33290)' => '33256', + 'Lüe (40210)' => '40163', + 'Lugaignac (33420)' => '33257', + 'Lugasson (33760)' => '33258', + 'Luglon (40630)' => '40165', + 'Lugon-et-l\'Île-du-Carnay (33240)' => '33259', + 'Lugos (33830)' => '33260', + 'Lunas (24130)' => '24246', + 'Lupersat (23190)' => '23113', + 'Lupsault (16140)' => '16194', + 'Luquet (65320)' => '65292', + 'Lurbe-Saint-Christau (64660)' => '64360', + 'Lusignac (24320)' => '24247', + 'Lusignan (86600)' => '86139', + 'Lusignan-Petit (47360)' => '47154', + 'Lussac (16450)' => '16195', + 'Lussac (17500)' => '17215', + 'Lussac (33570)' => '33261', + 'Lussac-les-Châteaux (86320)' => '86140', + 'Lussac-les-Églises (87360)' => '87087', + 'Lussagnet (40270)' => '40166', + 'Lussagnet-Lusson (64160)' => '64361', + 'Lussant (17430)' => '17216', + 'Lussas-et-Nontronneau (24300)' => '24248', + 'Lussat (23170)' => '23114', + 'Lusseray (79170)' => '79160', + 'Luxé (16230)' => '16196', + 'Luxe-Sumberraute (64120)' => '64362', + 'Luxey (40430)' => '40167', + 'Luzay (79100)' => '79161', + 'Lys (64260)' => '64363', + 'Macau (33460)' => '33262', + 'Macaye (64240)' => '64364', + 'Macqueville (17490)' => '17217', + 'Madaillan (47360)' => '47155', + 'Madirac (33670)' => '33263', + 'Madranges (19470)' => '19122', + 'Magescq (40140)' => '40168', + 'Magnac-Bourg (87380)' => '87088', + 'Magnac-Laval (87190)' => '87089', + 'Magnac-Lavalette-Villars (16320)' => '16198', + 'Magnac-sur-Touvre (16600)' => '16199', + 'Magnat-l\'Étrange (23260)' => '23115', + 'Magné (79460)' => '79162', + 'Magné (86160)' => '86141', + 'Mailhac-sur-Benaize (87160)' => '87090', + 'Maillas (40120)' => '40169', + 'Maillé (86190)' => '86142', + 'Maillères (40120)' => '40170', + 'Maine-de-Boixe (16230)' => '16200', + 'Mainsat (23700)' => '23116', + 'Mainxe (16200)' => '16202', + 'Mainzac (16380)' => '16203', + 'Mairé (86270)' => '86143', + 'Mairé-Levescault (79190)' => '79163', + 'Maison-Feyne (23800)' => '23117', + 'Maisonnais-sur-Tardoire (87440)' => '87091', + 'Maisonnay (79500)' => '79164', + 'Maisonneuve (86170)' => '86144', + 'Maisonnisses (23150)' => '23118', + 'Maisontiers (79600)' => '79165', + 'Malaussanne (64410)' => '64365', + 'Malaville (16120)' => '16204', + 'Malemort (19360)' => '19123', + 'Malleret (23260)' => '23119', + 'Malleret-Boussac (23600)' => '23120', + 'Malval (23220)' => '23121', + 'Manaurie (24620)' => '24249', + 'Mano (40410)' => '40171', + 'Manot (16500)' => '16205', + 'Mansac (19520)' => '19124', + 'Mansat-la-Courrière (23400)' => '23122', + 'Mansle (16230)' => '16206', + 'Mant (40700)' => '40172', + 'Manzac-sur-Vern (24110)' => '24251', + 'Marans (17230)' => '17218', + 'Maransin (33230)' => '33264', + 'Marc-la-Tour (19150)' => '19127', + 'Marçay (86370)' => '86145', + 'Marcellus (47200)' => '47156', + 'Marcenais (33620)' => '33266', + 'Marcheprime (33380)' => '33555', + 'Marcillac (33860)' => '33267', + 'Marcillac-la-Croisille (19320)' => '19125', + 'Marcillac-la-Croze (19500)' => '19126', + 'Marcillac-Lanville (16140)' => '16207', + 'Marcillac-Saint-Quentin (24200)' => '24252', + 'Marennes (17320)' => '17219', + 'Mareuil (16170)' => '16208', + 'Mareuil (24340)' => '24253', + 'Margaux (33460)' => '33268', + 'Margerides (19200)' => '19128', + 'Margueron (33220)' => '33269', + 'Marignac (17800)' => '17220', + 'Marigny (79360)' => '79166', + 'Marigny-Brizay (86380)' => '86146', + 'Marigny-Chemereau (86370)' => '86147', + 'Marillac-le-Franc (16110)' => '16209', + 'Marimbault (33430)' => '33270', + 'Marions (33690)' => '33271', + 'Marmande (47200)' => '47157', + 'Marmont-Pachas (47220)' => '47158', + 'Marnac (24220)' => '24254', + 'Marnay (86160)' => '86148', + 'Marnes (79600)' => '79167', + 'Marpaps (40330)' => '40173', + 'Marquay (24620)' => '24255', + 'Marsac (16570)' => '16210', + 'Marsac (23210)' => '23124', + 'Marsac-sur-l\'Isle (24430)' => '24256', + 'Marsais (17700)' => '17221', + 'Marsalès (24540)' => '24257', + 'Marsaneix (24750)' => '24258', + 'Marsas (33620)' => '33272', + 'Marsilly (17137)' => '17222', + 'Martaizé (86330)' => '86149', + 'Marthon (16380)' => '16211', + 'Martignas-sur-Jalle (33127)' => '33273', + 'Martillac (33650)' => '33274', + 'Martres (33760)' => '33275', + 'Marval (87440)' => '87092', + 'Masbaraud-Mérignat (23400)' => '23126', + 'Mascaraàs-Haron (64330)' => '64366', + 'Maslacq (64300)' => '64367', + 'Masléon (87130)' => '87093', + 'Masparraute (64120)' => '64368', + 'Maspie-Lalonquère-Juillacq (64350)' => '64369', + 'Masquières (47370)' => '47160', + 'Massac (17490)' => '17223', + 'Massais (79150)' => '79168', + 'Masseilles (33690)' => '33276', + 'Massels (47140)' => '47161', + 'Masseret (19510)' => '19129', + 'Massignac (16310)' => '16212', + 'Massognes (86170)' => '86150', + 'Massoulès (47140)' => '47162', + 'Massugas (33790)' => '33277', + 'Matha (17160)' => '17224', + 'Maucor (64160)' => '64370', + 'Maulay (86200)' => '86151', + 'Mauléon (79700)' => '79079', + 'Mauléon-Licharre (64130)' => '64371', + 'Mauprévoir (86460)' => '86152', + 'Maure (64460)' => '64372', + 'Maurens (24140)' => '24259', + 'Mauriac (33540)' => '33278', + 'Mauries (40320)' => '40174', + 'Maurrin (40270)' => '40175', + 'Maussac (19250)' => '19130', + 'Mautes (23190)' => '23127', + 'Mauvezin-d\'Armagnac (40240)' => '40176', + 'Mauvezin-sur-Gupie (47200)' => '47163', + 'Mauzac-et-Grand-Castang (24150)' => '24260', + 'Mauzé-sur-le-Mignon (79210)' => '79170', + 'Mauzé-Thouarsais (79100)' => '79171', + 'Mauzens-et-Miremont (24260)' => '24261', + 'Mayac (24420)' => '24262', + 'Maylis (40250)' => '40177', + 'Mazeirat (23150)' => '23128', + 'Mazeray (17400)' => '17226', + 'Mazères (33210)' => '33279', + 'Mazères-Lezons (64110)' => '64373', + 'Mazerolles (16310)' => '16213', + 'Mazerolles (17800)' => '17227', + 'Mazerolles (40090)' => '40178', + 'Mazerolles (64230)' => '64374', + 'Mazerolles (86320)' => '86153', + 'Mazeuil (86110)' => '86154', + 'Mazeyrolles (24550)' => '24263', + 'Mazières (16270)' => '16214', + 'Mazières-en-Gâtine (79310)' => '79172', + 'Mazières-Naresse (47210)' => '47164', + 'Mazières-sur-Béronne (79500)' => '79173', + 'Mazion (33390)' => '33280', + 'Méasnes (23360)' => '23130', + 'Médillac (16210)' => '16215', + 'Médis (17600)' => '17228', + 'Mées (40990)' => '40179', + 'Méharin (64120)' => '64375', + 'Meilhac (87800)' => '87094', + 'Meilhan (40400)' => '40180', + 'Meilhan-sur-Garonne (47180)' => '47165', + 'Meilhards (19510)' => '19131', + 'Meillon (64510)' => '64376', + 'Melle (79500)' => '79174', + 'Melleran (79190)' => '79175', + 'Mendionde (64240)' => '64377', + 'Menditte (64130)' => '64378', + 'Mendive (64220)' => '64379', + 'Ménesplet (24700)' => '24264', + 'Ménigoute (79340)' => '79176', + 'Ménoire (19190)' => '19132', + 'Mensignac (24350)' => '24266', + 'Méracq (64410)' => '64380', + 'Mercoeur (19430)' => '19133', + 'Mérignac (16200)' => '16216', + 'Mérignac (17210)' => '17229', + 'Mérignac (33700)' => '33281', + 'Mérignas (33350)' => '33282', + 'Mérinchal (23420)' => '23131', + 'Méritein (64190)' => '64381', + 'Merlines (19340)' => '19134', + 'Merpins (16100)' => '16217', + 'Meschers-sur-Gironde (17132)' => '17230', + 'Mescoules (24240)' => '24267', + 'Mesnac (16370)' => '16218', + 'Mesplède (64370)' => '64382', + 'Messac (17130)' => '17231', + 'Messanges (40660)' => '40181', + 'Messé (79120)' => '79177', + 'Messemé (86200)' => '86156', + 'Mesterrieux (33540)' => '33283', + 'Mestes (19200)' => '19135', + 'Meursac (17120)' => '17232', + 'Meux (17500)' => '17233', + 'Meuzac (87380)' => '87095', + 'Meymac (19250)' => '19136', + 'Meyrals (24220)' => '24268', + 'Meyrignac-l\'Église (19800)' => '19137', + 'Meyssac (19500)' => '19138', + 'Mézin (47170)' => '47167', + 'Mézos (40170)' => '40182', + 'Mialet (24450)' => '24269', + 'Mialos (64410)' => '64383', + 'Mignaloux-Beauvoir (86550)' => '86157', + 'Migné-Auxances (86440)' => '86158', + 'Migré (17330)' => '17234', + 'Migron (17770)' => '17235', + 'Milhac-d\'Auberoche (24330)' => '24270', + 'Milhac-de-Nontron (24470)' => '24271', + 'Millac (86150)' => '86159', + 'Millevaches (19290)' => '19139', + 'Mimbaste (40350)' => '40183', + 'Mimizan (40200)' => '40184', + 'Minzac (24610)' => '24272', + 'Mios (33380)' => '33284', + 'Miossens-Lanusse (64450)' => '64385', + 'Mirambeau (17150)' => '17236', + 'Miramont-de-Guyenne (47800)' => '47168', + 'Miramont-Sensacq (40320)' => '40185', + 'Mirebeau (86110)' => '86160', + 'Mirepeix (64800)' => '64386', + 'Missé (79100)' => '79178', + 'Misson (40290)' => '40186', + 'Moëze (17780)' => '17237', + 'Moirax (47310)' => '47169', + 'Moissannes (87400)' => '87099', + 'Molières (24480)' => '24273', + 'Moliets-et-Maa (40660)' => '40187', + 'Momas (64230)' => '64387', + 'Mombrier (33710)' => '33285', + 'Momuy (40700)' => '40188', + 'Momy (64350)' => '64388', + 'Monassut-Audiracq (64160)' => '64389', + 'Monbahus (47290)' => '47170', + 'Monbalen (47340)' => '47171', + 'Monbazillac (24240)' => '24274', + 'Moncaup (64350)' => '64390', + 'Moncaut (47310)' => '47172', + 'Moncayolle-Larrory-Mendibieu (64130)' => '64391', + 'Monceaux-sur-Dordogne (19400)' => '19140', + 'Moncla (64330)' => '64392', + 'Monclar (47380)' => '47173', + 'Moncontour (86330)' => '86161', + 'Moncoutant (79320)' => '79179', + 'Moncrabeau (47600)' => '47174', + 'Mondion (86230)' => '86162', + 'Monein (64360)' => '64393', + 'Monestier (24240)' => '24276', + 'Monestier-Merlines (19340)' => '19141', + 'Monestier-Port-Dieu (19110)' => '19142', + 'Monfaucon (24130)' => '24277', + 'Monflanquin (47150)' => '47175', + 'Mongaillard (47230)' => '47176', + 'Mongauzy (33190)' => '33287', + 'Monget (40700)' => '40189', + 'Monheurt (47160)' => '47177', + 'Monmadalès (24560)' => '24278', + 'Monmarvès (24560)' => '24279', + 'Monpazier (24540)' => '24280', + 'Monpezat (64350)' => '64394', + 'Monplaisant (24170)' => '24293', + 'Monprimblanc (33410)' => '33288', + 'Mons (16140)' => '16221', + 'Mons (17160)' => '17239', + 'Monsac (24440)' => '24281', + 'Monsaguel (24560)' => '24282', + 'Monsec (24340)' => '24283', + 'Monségur (33580)' => '33289', + 'Monségur (40700)' => '40190', + 'Monségur (47150)' => '47178', + 'Monségur (64460)' => '64395', + 'Monsempron-Libos (47500)' => '47179', + 'Mont (64300)' => '64396', + 'Mont-de-Marsan (40000)' => '40192', + 'Mont-Disse (64330)' => '64401', + 'Montagnac-d\'Auberoche (24210)' => '24284', + 'Montagnac-la-Crempse (24140)' => '24285', + 'Montagnac-sur-Auvignon (47600)' => '47180', + 'Montagnac-sur-Lède (47150)' => '47181', + 'Montagne (33570)' => '33290', + 'Montagoudin (33190)' => '33291', + 'Montagrier (24350)' => '24286', + 'Montagut (64410)' => '64397', + 'Montaignac-Saint-Hippolyte (19300)' => '19143', + 'Montaigut-le-Blanc (23320)' => '23132', + 'Montalembert (79190)' => '79180', + 'Montamisé (86360)' => '86163', + 'Montaner (64460)' => '64398', + 'Montardon (64121)' => '64399', + 'Montastruc (47380)' => '47182', + 'Montauriol (47330)' => '47183', + 'Montaut (24560)' => '24287', + 'Montaut (40500)' => '40191', + 'Montaut (47210)' => '47184', + 'Montaut (64800)' => '64400', + 'Montayral (47500)' => '47185', + 'Montazeau (24230)' => '24288', + 'Montboucher (23400)' => '23133', + 'Montboyer (16620)' => '16222', + 'Montbron (16220)' => '16223', + 'Montcaret (24230)' => '24289', + 'Montégut (40190)' => '40193', + 'Montemboeuf (16310)' => '16225', + 'Montendre (17130)' => '17240', + 'Montesquieu (47130)' => '47186', + 'Monteton (47120)' => '47187', + 'Montferrand-du-Périgord (24440)' => '24290', + 'Montfort (64190)' => '64403', + 'Montfort-en-Chalosse (40380)' => '40194', + 'Montgaillard (40500)' => '40195', + 'Montgibaud (19210)' => '19144', + 'Montguyon (17270)' => '17241', + 'Monthoiron (86210)' => '86164', + 'Montignac (24290)' => '24291', + 'Montignac (33760)' => '33292', + 'Montignac-Charente (16330)' => '16226', + 'Montignac-de-Lauzun (47800)' => '47188', + 'Montignac-le-Coq (16390)' => '16227', + 'Montignac-Toupinerie (47350)' => '47189', + 'Montigné (16170)' => '16228', + 'Montils (17800)' => '17242', + 'Montjean (16240)' => '16229', + 'Montlieu-la-Garde (17210)' => '17243', + 'Montmérac (16300)' => '16224', + 'Montmoreau-Saint-Cybard (16190)' => '16230', + 'Montmorillon (86500)' => '86165', + 'Montory (64470)' => '64404', + 'Montpellier-de-Médillan (17260)' => '17244', + 'Montpeyroux (24610)' => '24292', + 'Montpezat (47360)' => '47190', + 'Montpon-Ménestérol (24700)' => '24294', + 'Montpouillan (47200)' => '47191', + 'Montravers (79140)' => '79183', + 'Montrem (24110)' => '24295', + 'Montreuil-Bonnin (86470)' => '86166', + 'Montrol-Sénard (87330)' => '87100', + 'Montrollet (16420)' => '16231', + 'Montroy (17220)' => '17245', + 'Monts-sur-Guesnes (86420)' => '86167', + 'Montsoué (40500)' => '40196', + 'Montussan (33450)' => '33293', + 'Monviel (47290)' => '47192', + 'Moragne (17430)' => '17246', + 'Morcenx (40110)' => '40197', + 'Morganx (40700)' => '40198', + 'Morizès (33190)' => '33294', + 'Morlaàs (64160)' => '64405', + 'Morlanne (64370)' => '64406', + 'Mornac (16600)' => '16232', + 'Mornac-sur-Seudre (17113)' => '17247', + 'Mortagne-sur-Gironde (17120)' => '17248', + 'Mortemart (87330)' => '87101', + 'Mortiers (17500)' => '17249', + 'Morton (86120)' => '86169', + 'Mortroux (23220)' => '23136', + 'Mosnac (16120)' => '16233', + 'Mosnac (17240)' => '17250', + 'Mougon (79370)' => '79185', + 'Mouguerre (64990)' => '64407', + 'Mouhous (64330)' => '64408', + 'Mouillac (33240)' => '33295', + 'Mouleydier (24520)' => '24296', + 'Moulidars (16290)' => '16234', + 'Mouliets-et-Villemartin (33350)' => '33296', + 'Moulin-Neuf (24700)' => '24297', + 'Moulinet (47290)' => '47193', + 'Moulis-en-Médoc (33480)' => '33297', + 'Moulismes (86500)' => '86170', + 'Moulon (33420)' => '33298', + 'Moumour (64400)' => '64409', + 'Mourens (33410)' => '33299', + 'Mourenx (64150)' => '64410', + 'Mourioux-Vieilleville (23210)' => '23137', + 'Mouscardès (40290)' => '40199', + 'Moussac (86150)' => '86171', + 'Moustey (40410)' => '40200', + 'Moustier (47800)' => '47194', + 'Moustier-Ventadour (19300)' => '19145', + 'Mouterre-Silly (86200)' => '86173', + 'Mouterre-sur-Blourde (86430)' => '86172', + 'Mouthiers-sur-Boëme (16440)' => '16236', + 'Moutier-d\'Ahun (23150)' => '23138', + 'Moutier-Malcard (23220)' => '23139', + 'Moutier-Rozeille (23200)' => '23140', + 'Moutiers-sous-Chantemerle (79320)' => '79188', + 'Mouton (16460)' => '16237', + 'Moutonneau (16460)' => '16238', + 'Mouzon (16310)' => '16239', + 'Mugron (40250)' => '40201', + 'Muron (17430)' => '17253', + 'Musculdy (64130)' => '64411', + 'Mussidan (24400)' => '24299', + 'Nabas (64190)' => '64412', + 'Nabinaud (16390)' => '16240', + 'Nabirat (24250)' => '24300', + 'Nachamps (17380)' => '17254', + 'Nadaillac (24590)' => '24301', + 'Nailhac (24390)' => '24302', + 'Naillat (23800)' => '23141', + 'Naintré (86530)' => '86174', + 'Nalliers (86310)' => '86175', + 'Nanclars (16230)' => '16241', + 'Nancras (17600)' => '17255', + 'Nanteuil (79400)' => '79189', + 'Nanteuil-Auriac-de-Bourzac (24320)' => '24303', + 'Nanteuil-en-Vallée (16700)' => '16242', + 'Nantheuil (24800)' => '24304', + 'Nanthiat (24800)' => '24305', + 'Nantiat (87140)' => '87103', + 'Nantillé (17770)' => '17256', + 'Narcastet (64510)' => '64413', + 'Narp (64190)' => '64414', + 'Narrosse (40180)' => '40202', + 'Nassiet (40330)' => '40203', + 'Nastringues (24230)' => '24306', + 'Naujac-sur-Mer (33990)' => '33300', + 'Naujan-et-Postiac (33420)' => '33301', + 'Naussannes (24440)' => '24307', + 'Navailles-Angos (64450)' => '64415', + 'Navarrenx (64190)' => '64416', + 'Naves (19460)' => '19146', + 'Nay (64800)' => '64417', + 'Néac (33500)' => '33302', + 'Nedde (87120)' => '87104', + 'Négrondes (24460)' => '24308', + 'Néoux (23200)' => '23142', + 'Nérac (47600)' => '47195', + 'Nerbis (40250)' => '40204', + 'Nercillac (16200)' => '16243', + 'Néré (17510)' => '17257', + 'Nérigean (33750)' => '33303', + 'Nérignac (86150)' => '86176', + 'Nersac (16440)' => '16244', + 'Nespouls (19600)' => '19147', + 'Neuffons (33580)' => '33304', + 'Neuillac (17520)' => '17258', + 'Neulles (17500)' => '17259', + 'Neuvic (19160)' => '19148', + 'Neuvic (24190)' => '24309', + 'Neuvic-Entier (87130)' => '87105', + 'Neuvicq (17270)' => '17260', + 'Neuvicq-le-Château (17490)' => '17261', + 'Neuville (19380)' => '19149', + 'Neuville-de-Poitou (86170)' => '86177', + 'Neuvy-Bouin (79130)' => '79190', + 'Nexon (87800)' => '87106', + 'Nicole (47190)' => '47196', + 'Nieuil (16270)' => '16245', + 'Nieuil-l\'Espoir (86340)' => '86178', + 'Nieul (87510)' => '87107', + 'Nieul-le-Virouil (17150)' => '17263', + 'Nieul-lès-Saintes (17810)' => '17262', + 'Nieul-sur-Mer (17137)' => '17264', + 'Nieulle-sur-Seudre (17600)' => '17265', + 'Niort (79000)' => '79191', + 'Noailhac (19500)' => '19150', + 'Noaillac (33190)' => '33306', + 'Noaillan (33730)' => '33307', + 'Noailles (19600)' => '19151', + 'Noguères (64150)' => '64418', + 'Nomdieu (47600)' => '47197', + 'Nonac (16190)' => '16246', + 'Nonards (19120)' => '19152', + 'Nonaville (16120)' => '16247', + 'Nontron (24300)' => '24311', + 'Noth (23300)' => '23143', + 'Notre-Dame-de-Sanilhac (24660)' => '24312', + 'Nouaillé-Maupertuis (86340)' => '86180', + 'Nouhant (23170)' => '23145', + 'Nouic (87330)' => '87108', + 'Nousse (40380)' => '40205', + 'Nousty (64420)' => '64419', + 'Nouzerines (23600)' => '23146', + 'Nouzerolles (23360)' => '23147', + 'Nouziers (23350)' => '23148', + 'Nuaillé-d\'Aunis (17540)' => '17267', + 'Nuaillé-sur-Boutonne (17470)' => '17268', + 'Nueil-les-Aubiers (79250)' => '79195', + 'Nueil-sous-Faye (86200)' => '86181', + 'Objat (19130)' => '19153', + 'Oeyregave (40300)' => '40206', + 'Oeyreluy (40180)' => '40207', + 'Ogenne-Camptort (64190)' => '64420', + 'Ogeu-les-Bains (64680)' => '64421', + 'Oiron (79100)' => '79196', + 'Oloron-Sainte-Marie (64400)' => '64422', + 'Omet (33410)' => '33308', + 'Onard (40380)' => '40208', + 'Ondres (40440)' => '40209', + 'Onesse-Laharie (40110)' => '40210', + 'Oraàs (64390)' => '64423', + 'Oradour (16140)' => '16248', + 'Oradour-Fanais (16500)' => '16249', + 'Oradour-Saint-Genest (87210)' => '87109', + 'Oradour-sur-Glane (87520)' => '87110', + 'Oradour-sur-Vayres (87150)' => '87111', + 'Orches (86230)' => '86182', + 'Ordiarp (64130)' => '64424', + 'Ordonnac (33340)' => '33309', + 'Orègue (64120)' => '64425', + 'Orgedeuil (16220)' => '16250', + 'Orgnac-sur-Vézère (19410)' => '19154', + 'Origne (33113)' => '33310', + 'Orignolles (17210)' => '17269', + 'Orin (64400)' => '64426', + 'Oriolles (16480)' => '16251', + 'Orion (64390)' => '64427', + 'Orist (40300)' => '40211', + 'Orival (16210)' => '16252', + 'Orliac (24170)' => '24313', + 'Orliac-de-Bar (19390)' => '19155', + 'Orliaguet (24370)' => '24314', + 'Oroux (79390)' => '79197', + 'Orriule (64390)' => '64428', + 'Orsanco (64120)' => '64429', + 'Orthevielle (40300)' => '40212', + 'Orthez (64300)' => '64430', + 'Orx (40230)' => '40213', + 'Os-Marsillon (64150)' => '64431', + 'Ossages (40290)' => '40214', + 'Ossas-Suhare (64470)' => '64432', + 'Osse-en-Aspe (64490)' => '64433', + 'Ossenx (64190)' => '64434', + 'Osserain-Rivareyte (64390)' => '64435', + 'Ossès (64780)' => '64436', + 'Ostabat-Asme (64120)' => '64437', + 'Ouillon (64160)' => '64438', + 'Ousse (64320)' => '64439', + 'Ousse-Suzan (40110)' => '40215', + 'Ouzilly (86380)' => '86184', + 'Oyré (86220)' => '86186', + 'Ozenx-Montestrucq (64300)' => '64440', + 'Ozillac (17500)' => '17270', + 'Ozourt (40380)' => '40216', + 'Pageas (87230)' => '87112', + 'Pagolle (64120)' => '64441', + 'Paillé (17470)' => '17271', + 'Paillet (33550)' => '33311', + 'Pailloles (47440)' => '47198', + 'Paizay-le-Chapt (79170)' => '79198', + 'Paizay-le-Sec (86300)' => '86187', + 'Paizay-le-Tort (79500)' => '79199', + 'Paizay-Naudouin-Embourie (16240)' => '16253', + 'Palazinges (19190)' => '19156', + 'Palisse (19160)' => '19157', + 'Palluaud (16390)' => '16254', + 'Pamplie (79220)' => '79200', + 'Pamproux (79800)' => '79201', + 'Panazol (87350)' => '87114', + 'Pandrignes (19150)' => '19158', + 'Parbayse (64360)' => '64442', + 'Parcoul-Chenaud (24410)' => '24316', + 'Pardaillan (47120)' => '47199', + 'Pardies (64150)' => '64443', + 'Pardies-Piétat (64800)' => '64444', + 'Parempuyre (33290)' => '33312', + 'Parentis-en-Born (40160)' => '40217', + 'Parleboscq (40310)' => '40218', + 'Parranquet (47210)' => '47200', + 'Parsac-Rimondeix (23140)' => '23149', + 'Parthenay (79200)' => '79202', + 'Parzac (16450)' => '16255', + 'Pas-de-Jeu (79100)' => '79203', + 'Passirac (16480)' => '16256', + 'Pau (64000)' => '64445', + 'Pauillac (33250)' => '33314', + 'Paulhiac (47150)' => '47202', + 'Paulin (24590)' => '24317', + 'Paunat (24510)' => '24318', + 'Paussac-et-Saint-Vivien (24310)' => '24319', + 'Payré (86700)' => '86188', + 'Payros-Cazautets (40320)' => '40219', + 'Payroux (86350)' => '86189', + 'Pays de Belvès (24170)' => '24035', + 'Payzac (24270)' => '24320', + 'Pazayac (24120)' => '24321', + 'Pécorade (40320)' => '40220', + 'Pellegrue (33790)' => '33316', + 'Penne-d\'Agenais (47140)' => '47203', + 'Pensol (87440)' => '87115', + 'Péré (17700)' => '17272', + 'Péret-Bel-Air (19300)' => '19159', + 'Pérignac (16250)' => '16258', + 'Pérignac (17800)' => '17273', + 'Périgné (79170)' => '79204', + 'Périgny (17180)' => '17274', + 'Périgueux (24000)' => '24322', + 'Périssac (33240)' => '33317', + 'Pérols-sur-Vézère (19170)' => '19160', + 'Perpezac-le-Blanc (19310)' => '19161', + 'Perpezac-le-Noir (19410)' => '19162', + 'Perquie (40190)' => '40221', + 'Pers (79190)' => '79205', + 'Persac (86320)' => '86190', + 'Pessac (33600)' => '33318', + 'Pessac-sur-Dordogne (33890)' => '33319', + 'Pessines (17810)' => '17275', + 'Petit-Bersac (24600)' => '24323', + 'Petit-Palais-et-Cornemps (33570)' => '33320', + 'Peujard (33240)' => '33321', + 'Pey (40300)' => '40222', + 'Peyrabout (23000)' => '23150', + 'Peyrat-de-Bellac (87300)' => '87116', + 'Peyrat-la-Nonière (23130)' => '23151', + 'Peyrat-le-Château (87470)' => '87117', + 'Peyre (40700)' => '40223', + 'Peyrehorade (40300)' => '40224', + 'Peyrelevade (19290)' => '19164', + 'Peyrelongue-Abos (64350)' => '64446', + 'Peyrière (47350)' => '47204', + 'Peyrignac (24210)' => '24324', + 'Peyrilhac (87510)' => '87118', + 'Peyrillac-et-Millac (24370)' => '24325', + 'Peyrissac (19260)' => '19165', + 'Peyzac-le-Moustier (24620)' => '24326', + 'Pezuls (24510)' => '24327', + 'Philondenx (40320)' => '40225', + 'Piégut-Pluviers (24360)' => '24328', + 'Pierre-Buffière (87260)' => '87119', + 'Pierrefitte (19450)' => '19166', + 'Pierrefitte (23130)' => '23152', + 'Pierrefitte (79330)' => '79209', + 'Piets-Plasence-Moustrou (64410)' => '64447', + 'Pillac (16390)' => '16260', + 'Pimbo (40320)' => '40226', + 'Pindères (47700)' => '47205', + 'Pindray (86500)' => '86191', + 'Pinel-Hauterive (47380)' => '47206', + 'Pineuilh (33220)' => '33324', + 'Pionnat (23140)' => '23154', + 'Pioussay (79110)' => '79211', + 'Pisany (17600)' => '17278', + 'Pissos (40410)' => '40227', + 'Plaisance (24560)' => '24168', + 'Plaisance (86500)' => '86192', + 'Plassac (17240)' => '17279', + 'Plassac (33390)' => '33325', + 'Plassac-Rouffiac (16250)' => '16263', + 'Plassay (17250)' => '17280', + 'Plazac (24580)' => '24330', + 'Pleine-Selve (33820)' => '33326', + 'Pleumartin (86450)' => '86193', + 'Pleuville (16490)' => '16264', + 'Pliboux (79190)' => '79212', + 'Podensac (33720)' => '33327', + 'Poey-d\'Oloron (64400)' => '64449', + 'Poey-de-Lescar (64230)' => '64448', + 'Poitiers (86000)' => '86194', + 'Polignac (17210)' => '17281', + 'Pomarez (40360)' => '40228', + 'Pomerol (33500)' => '33328', + 'Pommiers-Moulons (17130)' => '17282', + 'Pompaire (79200)' => '79213', + 'Pompéjac (33730)' => '33329', + 'Pompiey (47230)' => '47207', + 'Pompignac (33370)' => '33330', + 'Pompogne (47420)' => '47208', + 'Pomport (24240)' => '24331', + 'Pomps (64370)' => '64450', + 'Pondaurat (33190)' => '33331', + 'Pons (17800)' => '17283', + 'Ponson-Debat-Pouts (64460)' => '64451', + 'Ponson-Dessus (64460)' => '64452', + 'Pont-du-Casse (47480)' => '47209', + 'Pont-l\'Abbé-d\'Arnoult (17250)' => '17284', + 'Pontacq (64530)' => '64453', + 'Pontarion (23250)' => '23155', + 'Pontcharraud (23260)' => '23156', + 'Pontenx-les-Forges (40200)' => '40229', + 'Ponteyraud (24410)' => '24333', + 'Pontiacq-Viellepinte (64460)' => '64454', + 'Pontonx-sur-l\'Adour (40465)' => '40230', + 'Pontours (24150)' => '24334', + 'Porchères (33660)' => '33332', + 'Port-d\'Envaux (17350)' => '17285', + 'Port-de-Lanne (40300)' => '40231', + 'Port-de-Piles (86220)' => '86195', + 'Port-des-Barques (17730)' => '17484', + 'Port-Sainte-Foy-et-Ponchapt (33220)' => '24335', + 'Port-Sainte-Marie (47130)' => '47210', + 'Portet (64330)' => '64455', + 'Portets (33640)' => '33334', + 'Pouançay (86120)' => '86196', + 'Pouant (86200)' => '86197', + 'Poudenas (47170)' => '47211', + 'Poudenx (40700)' => '40232', + 'Pouffonds (79500)' => '79214', + 'Pougne-Hérisson (79130)' => '79215', + 'Pouillac (17210)' => '17287', + 'Pouillé (86800)' => '86198', + 'Pouillon (40350)' => '40233', + 'Pouliacq (64410)' => '64456', + 'Poullignac (16190)' => '16267', + 'Poursac (16700)' => '16268', + 'Poursay-Garnaud (17400)' => '17288', + 'Poursiugues-Boucoue (64410)' => '64457', + 'Poussanges (23500)' => '23158', + 'Poussignac (47700)' => '47212', + 'Pouydesseaux (40120)' => '40234', + 'Poyanne (40380)' => '40235', + 'Poyartin (40380)' => '40236', + 'Pradines (19170)' => '19168', + 'Prahecq (79230)' => '79216', + 'Prailles (79370)' => '79217', + 'Pranzac (16110)' => '16269', + 'Prats-de-Carlux (24370)' => '24336', + 'Prats-du-Périgord (24550)' => '24337', + 'Prayssas (47360)' => '47213', + 'Préchac (33730)' => '33336', + 'Préchacq-Josbaig (64190)' => '64458', + 'Préchacq-les-Bains (40465)' => '40237', + 'Préchacq-Navarrenx (64190)' => '64459', + 'Précilhon (64400)' => '64460', + 'Préguillac (17460)' => '17289', + 'Preignac (33210)' => '33337', + 'Pressac (86460)' => '86200', + 'Pressignac (16150)' => '16270', + 'Pressignac-Vicq (24150)' => '24338', + 'Pressigny (79390)' => '79218', + 'Preyssac-d\'Excideuil (24160)' => '24339', + 'Priaires (79210)' => '79219', + 'Prignac (17160)' => '17290', + 'Prignac-en-Médoc (33340)' => '33338', + 'Prignac-et-Marcamps (33710)' => '33339', + 'Prigonrieux (24130)' => '24340', + 'Prin-Deyrançon (79210)' => '79220', + 'Prinçay (86420)' => '86201', + 'Prissé-la-Charrière (79360)' => '79078', + 'Proissans (24200)' => '24341', + 'Puch-d\'Agenais (47160)' => '47214', + 'Pugnac (33710)' => '33341', + 'Pugny (79320)' => '79222', + 'Puihardy (79160)' => '79223', + 'Puilboreau (17138)' => '17291', + 'Puisseguin (33570)' => '33342', + 'Pujo-le-Plan (40190)' => '40238', + 'Pujols (33350)' => '33344', + 'Pujols (47300)' => '47215', + 'Pujols-sur-Ciron (33210)' => '33343', + 'Puy-d\'Arnac (19120)' => '19169', + 'Puy-du-Lac (17380)' => '17292', + 'Puy-Malsignat (23130)' => '23159', + 'Puybarban (33190)' => '33346', + 'Puymiclan (47350)' => '47216', + 'Puymirol (47270)' => '47217', + 'Puymoyen (16400)' => '16271', + 'Puynormand (33660)' => '33347', + 'Puyol-Cazalet (40320)' => '40239', + 'Puyoô (64270)' => '64461', + 'Puyravault (17700)' => '17293', + 'Puyréaux (16230)' => '16272', + 'Puyrenier (24340)' => '24344', + 'Puyrolland (17380)' => '17294', + 'Puysserampion (47800)' => '47218', + 'Queaux (86150)' => '86203', + 'Queyrac (33340)' => '33348', + 'Queyssac (24140)' => '24345', + 'Queyssac-les-Vignes (19120)' => '19170', + 'Quinçay (86190)' => '86204', + 'Quinsac (24530)' => '24346', + 'Quinsac (33360)' => '33349', + 'Raix (16240)' => '16273', + 'Ramous (64270)' => '64462', + 'Rampieux (24440)' => '24347', + 'Rancogne (16110)' => '16274', + 'Rancon (87290)' => '87121', + 'Ranton (86200)' => '86205', + 'Ranville-Breuillaud (16140)' => '16275', + 'Raslay (86120)' => '86206', + 'Rauzan (33420)' => '33350', + 'Rayet (47210)' => '47219', + 'Razac-d\'Eymet (24500)' => '24348', + 'Razac-de-Saussignac (24240)' => '24349', + 'Razac-sur-l\'Isle (24430)' => '24350', + 'Razès (87640)' => '87122', + 'Razimet (47160)' => '47220', + 'Réaup-Lisse (47170)' => '47221', + 'Réaux sur Trèfle (17500)' => '17295', + 'Rébénacq (64260)' => '64463', + 'Reffannes (79420)' => '79225', + 'Reignac (16360)' => '16276', + 'Reignac (33860)' => '33351', + 'Rempnat (87120)' => '87123', + 'Renung (40270)' => '40240', + 'Réparsac (16200)' => '16277', + 'Rétaud (17460)' => '17296', + 'Reterre (23110)' => '23160', + 'Retjons (40120)' => '40164', + 'Reygade (19430)' => '19171', + 'Ribagnac (24240)' => '24351', + 'Ribarrouy (64330)' => '64464', + 'Ribérac (24600)' => '24352', + 'Rilhac-Lastours (87800)' => '87124', + 'Rilhac-Rancon (87570)' => '87125', + 'Rilhac-Treignac (19260)' => '19172', + 'Rilhac-Xaintrie (19220)' => '19173', + 'Rimbez-et-Baudiets (40310)' => '40242', + 'Rimons (33580)' => '33353', + 'Riocaud (33220)' => '33354', + 'Rion-des-Landes (40370)' => '40243', + 'Rions (33410)' => '33355', + 'Rioux (17460)' => '17298', + 'Rioux-Martin (16210)' => '16279', + 'Riupeyrous (64160)' => '64465', + 'Rivedoux-Plage (17940)' => '17297', + 'Rivehaute (64190)' => '64466', + 'Rives (47210)' => '47223', + 'Rivière-Saas-et-Gourby (40180)' => '40244', + 'Rivières (16110)' => '16280', + 'Roaillan (33210)' => '33357', + 'Roche-le-Peyroux (19160)' => '19175', + 'Rochechouart (87600)' => '87126', + 'Rochefort (17300)' => '17299', + 'Roches (23270)' => '23162', + 'Roches-Prémarie-Andillé (86340)' => '86209', + 'Roiffé (86120)' => '86210', + 'Rom (79120)' => '79230', + 'Romagne (33760)' => '33358', + 'Romagne (86700)' => '86211', + 'Romans (79260)' => '79231', + 'Romazières (17510)' => '17301', + 'Romegoux (17250)' => '17302', + 'Romestaing (47250)' => '47224', + 'Ronsenac (16320)' => '16283', + 'Rontignon (64110)' => '64467', + 'Roquebrune (33580)' => '33359', + 'Roquefort (40120)' => '40245', + 'Roquefort (47310)' => '47225', + 'Roquiague (64130)' => '64468', + 'Rosiers-d\'Égletons (19300)' => '19176', + 'Rosiers-de-Juillac (19350)' => '19177', + 'Rouffiac (16210)' => '16284', + 'Rouffiac (17800)' => '17304', + 'Rouffignac (17130)' => '17305', + 'Rouffignac-de-Sigoulès (24240)' => '24357', + 'Rouffignac-Saint-Cernin-de-Reilhac (24580)' => '24356', + 'Rougnac (16320)' => '16285', + 'Rougnat (23700)' => '23164', + 'Rouillac (16170)' => '16286', + 'Rouillé (86480)' => '86213', + 'Roullet-Saint-Estèphe (16440)' => '16287', + 'Roumagne (47800)' => '47226', + 'Roumazières-Loubert (16270)' => '16192', + 'Roussac (87140)' => '87128', + 'Roussines (16310)' => '16289', + 'Rouzède (16220)' => '16290', + 'Royan (17200)' => '17306', + 'Royère-de-Vassivière (23460)' => '23165', + 'Royères (87400)' => '87129', + 'Roziers-Saint-Georges (87130)' => '87130', + 'Ruch (33350)' => '33361', + 'Rudeau-Ladosse (24340)' => '24221', + 'Ruelle-sur-Touvre (16600)' => '16291', + 'Ruffec (16700)' => '16292', + 'Ruffiac (47700)' => '47227', + 'Sablonceaux (17600)' => '17307', + 'Sablons (33910)' => '33362', + 'Sabres (40630)' => '40246', + 'Sadillac (24500)' => '24359', + 'Sadirac (33670)' => '33363', + 'Sadroc (19270)' => '19178', + 'Sagelat (24170)' => '24360', + 'Sagnat (23800)' => '23166', + 'Saillac (19500)' => '19179', + 'Saillans (33141)' => '33364', + 'Saillat-sur-Vienne (87720)' => '87131', + 'Saint Aulaye-Puymangou (24410)' => '24376', + 'Saint Maurice Étusson (79150)' => '79280', + 'Saint-Abit (64800)' => '64469', + 'Saint-Adjutory (16310)' => '16293', + 'Saint-Agnant (17620)' => '17308', + 'Saint-Agnant-de-Versillat (23300)' => '23177', + 'Saint-Agnant-près-Crocq (23260)' => '23178', + 'Saint-Agne (24520)' => '24361', + 'Saint-Agnet (40800)' => '40247', + 'Saint-Aignan (33126)' => '33365', + 'Saint-Aigulin (17360)' => '17309', + 'Saint-Alpinien (23200)' => '23179', + 'Saint-Amand (23200)' => '23180', + 'Saint-Amand-de-Coly (24290)' => '24364', + 'Saint-Amand-de-Vergt (24380)' => '24365', + 'Saint-Amand-Jartoudeix (23400)' => '23181', + 'Saint-Amand-le-Petit (87120)' => '87132', + 'Saint-Amand-Magnazeix (87290)' => '87133', + 'Saint-Amand-sur-Sèvre (79700)' => '79235', + 'Saint-Amant-de-Boixe (16330)' => '16295', + 'Saint-Amant-de-Bonnieure (16230)' => '16296', + 'Saint-Amant-de-Montmoreau (16190)' => '16294', + 'Saint-Amant-de-Nouère (16170)' => '16298', + 'Saint-André-d\'Allas (24200)' => '24366', + 'Saint-André-de-Cubzac (33240)' => '33366', + 'Saint-André-de-Double (24190)' => '24367', + 'Saint-André-de-Lidon (17260)' => '17310', + 'Saint-André-de-Seignanx (40390)' => '40248', + 'Saint-André-du-Bois (33490)' => '33367', + 'Saint-André-et-Appelles (33220)' => '33369', + 'Saint-André-sur-Sèvre (79380)' => '79236', + 'Saint-Androny (33390)' => '33370', + 'Saint-Angeau (16230)' => '16300', + 'Saint-Angel (19200)' => '19180', + 'Saint-Antoine-Cumond (24410)' => '24368', + 'Saint-Antoine-d\'Auberoche (24330)' => '24369', + 'Saint-Antoine-de-Breuilh (24230)' => '24370', + 'Saint-Antoine-de-Ficalba (47340)' => '47228', + 'Saint-Antoine-du-Queyret (33790)' => '33372', + 'Saint-Antoine-sur-l\'Isle (33660)' => '33373', + 'Saint-Aquilin (24110)' => '24371', + 'Saint-Armou (64160)' => '64470', + 'Saint-Astier (24110)' => '24372', + 'Saint-Astier (47120)' => '47229', + 'Saint-Aubin (40250)' => '40249', + 'Saint-Aubin (47150)' => '47230', + 'Saint-Aubin-de-Blaye (33820)' => '33374', + 'Saint-Aubin-de-Branne (33420)' => '33375', + 'Saint-Aubin-de-Cadelech (24500)' => '24373', + 'Saint-Aubin-de-Lanquais (24560)' => '24374', + 'Saint-Aubin-de-Médoc (33160)' => '33376', + 'Saint-Aubin-de-Nabirat (24250)' => '24375', + 'Saint-Aubin-du-Plain (79300)' => '79238', + 'Saint-Aubin-le-Cloud (79450)' => '79239', + 'Saint-Augustin (17570)' => '17311', + 'Saint-Augustin (19390)' => '19181', + 'Saint-Aulaire (19130)' => '19182', + 'Saint-Aulais-la-Chapelle (16300)' => '16301', + 'Saint-Auvent (87310)' => '87135', + 'Saint-Avit (16210)' => '16302', + 'Saint-Avit (40090)' => '40250', + 'Saint-Avit (47350)' => '47231', + 'Saint-Avit-de-Soulège (33220)' => '33377', + 'Saint-Avit-de-Tardes (23200)' => '23182', + 'Saint-Avit-de-Vialard (24260)' => '24377', + 'Saint-Avit-le-Pauvre (23480)' => '23183', + 'Saint-Avit-Rivière (24540)' => '24378', + 'Saint-Avit-Saint-Nazaire (33220)' => '33378', + 'Saint-Avit-Sénieur (24440)' => '24379', + 'Saint-Barbant (87330)' => '87136', + 'Saint-Bard (23260)' => '23184', + 'Saint-Barthélemy (40390)' => '40251', + 'Saint-Barthélemy-d\'Agenais (47350)' => '47232', + 'Saint-Barthélemy-de-Bellegarde (24700)' => '24380', + 'Saint-Barthélemy-de-Bussière (24360)' => '24381', + 'Saint-Bazile (87150)' => '87137', + 'Saint-Bazile-de-la-Roche (19320)' => '19183', + 'Saint-Bazile-de-Meyssac (19500)' => '19184', + 'Saint-Benoît (86280)' => '86214', + 'Saint-Boès (64300)' => '64471', + 'Saint-Bonnet (16300)' => '16303', + 'Saint-Bonnet-Avalouze (19150)' => '19185', + 'Saint-Bonnet-Briance (87260)' => '87138', + 'Saint-Bonnet-de-Bellac (87300)' => '87139', + 'Saint-Bonnet-Elvert (19380)' => '19186', + 'Saint-Bonnet-l\'Enfantier (19410)' => '19188', + 'Saint-Bonnet-la-Rivière (19130)' => '19187', + 'Saint-Bonnet-les-Tours-de-Merle (19430)' => '19189', + 'Saint-Bonnet-près-Bort (19200)' => '19190', + 'Saint-Bonnet-sur-Gironde (17150)' => '17312', + 'Saint-Brice (16100)' => '16304', + 'Saint-Brice (33540)' => '33379', + 'Saint-Brice-sur-Vienne (87200)' => '87140', + 'Saint-Bris-des-Bois (17770)' => '17313', + 'Saint-Caprais-de-Blaye (33820)' => '33380', + 'Saint-Caprais-de-Bordeaux (33880)' => '33381', + 'Saint-Caprais-de-Lerm (47270)' => '47234', + 'Saint-Capraise-d\'Eymet (24500)' => '24383', + 'Saint-Capraise-de-Lalinde (24150)' => '24382', + 'Saint-Cassien (24540)' => '24384', + 'Saint-Castin (64160)' => '64472', + 'Saint-Cernin-de-l\'Herm (24550)' => '24386', + 'Saint-Cernin-de-Labarde (24560)' => '24385', + 'Saint-Cernin-de-Larche (19600)' => '19191', + 'Saint-Césaire (17770)' => '17314', + 'Saint-Chabrais (23130)' => '23185', + 'Saint-Chamant (19380)' => '19192', + 'Saint-Chamassy (24260)' => '24388', + 'Saint-Christoly-de-Blaye (33920)' => '33382', + 'Saint-Christoly-Médoc (33340)' => '33383', + 'Saint-Christophe (16420)' => '16306', + 'Saint-Christophe (17220)' => '17315', + 'Saint-Christophe (23000)' => '23186', + 'Saint-Christophe (86230)' => '86217', + 'Saint-Christophe-de-Double (33230)' => '33385', + 'Saint-Christophe-des-Bardes (33330)' => '33384', + 'Saint-Christophe-sur-Roc (79220)' => '79241', + 'Saint-Cibard (33570)' => '33386', + 'Saint-Ciers-Champagne (17520)' => '17316', + 'Saint-Ciers-d\'Abzac (33910)' => '33387', + 'Saint-Ciers-de-Canesse (33710)' => '33388', + 'Saint-Ciers-du-Taillon (17240)' => '17317', + 'Saint-Ciers-sur-Bonnieure (16230)' => '16307', + 'Saint-Ciers-sur-Gironde (33820)' => '33389', + 'Saint-Cirgues-la-Loutre (19220)' => '19193', + 'Saint-Cirq (24260)' => '24389', + 'Saint-Clair (86330)' => '86218', + 'Saint-Claud (16450)' => '16308', + 'Saint-Clément (19700)' => '19194', + 'Saint-Clément-des-Baleines (17590)' => '17318', + 'Saint-Colomb-de-Lauzun (47410)' => '47235', + 'Saint-Côme (33430)' => '33391', + 'Saint-Coutant (16350)' => '16310', + 'Saint-Coutant (79120)' => '79243', + 'Saint-Coutant-le-Grand (17430)' => '17320', + 'Saint-Crépin (17380)' => '17321', + 'Saint-Crépin-d\'Auberoche (24330)' => '24390', + 'Saint-Crépin-de-Richemont (24310)' => '24391', + 'Saint-Crépin-et-Carlucet (24590)' => '24392', + 'Saint-Cricq-Chalosse (40700)' => '40253', + 'Saint-Cricq-du-Gave (40300)' => '40254', + 'Saint-Cricq-Villeneuve (40190)' => '40255', + 'Saint-Cybardeaux (16170)' => '16312', + 'Saint-Cybranet (24250)' => '24395', + 'Saint-Cyprien (19130)' => '19195', + 'Saint-Cyprien (24220)' => '24396', + 'Saint-Cyr (86130)' => '86219', + 'Saint-Cyr (87310)' => '87141', + 'Saint-Cyr-du-Doret (17170)' => '17322', + 'Saint-Cyr-la-Lande (79100)' => '79244', + 'Saint-Cyr-la-Roche (19130)' => '19196', + 'Saint-Cyr-les-Champagnes (24270)' => '24397', + 'Saint-Denis-d\'Oléron (17650)' => '17323', + 'Saint-Denis-de-Pile (33910)' => '33393', + 'Saint-Denis-des-Murs (87400)' => '87142', + 'Saint-Dizant-du-Bois (17150)' => '17324', + 'Saint-Dizant-du-Gua (17240)' => '17325', + 'Saint-Dizier-la-Tour (23130)' => '23187', + 'Saint-Dizier-les-Domaines (23270)' => '23188', + 'Saint-Dizier-Leyrenne (23400)' => '23189', + 'Saint-Domet (23190)' => '23190', + 'Saint-Dos (64270)' => '64474', + 'Saint-Éloi (23000)' => '23191', + 'Saint-Éloy-les-Tuileries (19210)' => '19198', + 'Saint-Émilion (33330)' => '33394', + 'Saint-Esteben (64640)' => '64476', + 'Saint-Estèphe (24360)' => '24398', + 'Saint-Estèphe (33180)' => '33395', + 'Saint-Étienne-aux-Clos (19200)' => '19199', + 'Saint-Étienne-d\'Orthe (40300)' => '40256', + 'Saint-Étienne-de-Baïgorry (64430)' => '64477', + 'Saint-Étienne-de-Fougères (47380)' => '47239', + 'Saint-Étienne-de-Fursac (23290)' => '23192', + 'Saint-Étienne-de-Lisse (33330)' => '33396', + 'Saint-Étienne-de-Puycorbier (24400)' => '24399', + 'Saint-Étienne-de-Villeréal (47210)' => '47240', + 'Saint-Étienne-la-Cigogne (79360)' => '79247', + 'Saint-Étienne-la-Geneste (19160)' => '19200', + 'Saint-Eugène (17520)' => '17326', + 'Saint-Eutrope (16190)' => '16314', + 'Saint-Eutrope-de-Born (47210)' => '47241', + 'Saint-Exupéry (33190)' => '33398', + 'Saint-Exupéry-les-Roches (19200)' => '19201', + 'Saint-Faust (64110)' => '64478', + 'Saint-Félix (16480)' => '16315', + 'Saint-Félix (17330)' => '17327', + 'Saint-Félix-de-Bourdeilles (24340)' => '24403', + 'Saint-Félix-de-Foncaude (33540)' => '33399', + 'Saint-Félix-de-Reillac-et-Mortemart (24260)' => '24404', + 'Saint-Félix-de-Villadeix (24510)' => '24405', + 'Saint-Ferme (33580)' => '33400', + 'Saint-Fiel (23000)' => '23195', + 'Saint-Fort-sur-Gironde (17240)' => '17328', + 'Saint-Fort-sur-le-Né (16130)' => '16316', + 'Saint-Fraigne (16140)' => '16317', + 'Saint-Fréjoux (19200)' => '19204', + 'Saint-Frion (23500)' => '23196', + 'Saint-Front (16460)' => '16318', + 'Saint-Front-d\'Alemps (24460)' => '24408', + 'Saint-Front-de-Pradoux (24400)' => '24409', + 'Saint-Front-la-Rivière (24300)' => '24410', + 'Saint-Front-sur-Lémance (47500)' => '47242', + 'Saint-Front-sur-Nizonne (24300)' => '24411', + 'Saint-Froult (17780)' => '17329', + 'Saint-Gaudent (86400)' => '86220', + 'Saint-Gein (40190)' => '40259', + 'Saint-Gelais (79410)' => '79249', + 'Saint-Génard (79500)' => '79251', + 'Saint-Gence (87510)' => '87143', + 'Saint-Généroux (79600)' => '79252', + 'Saint-Genès-de-Blaye (33390)' => '33405', + 'Saint-Genès-de-Castillon (33350)' => '33406', + 'Saint-Genès-de-Fronsac (33240)' => '33407', + 'Saint-Genès-de-Lombaud (33670)' => '33408', + 'Saint-Genest-d\'Ambière (86140)' => '86221', + 'Saint-Genest-sur-Roselle (87260)' => '87144', + 'Saint-Geniès (24590)' => '24412', + 'Saint-Geniez-ô-Merle (19220)' => '19205', + 'Saint-Genis-d\'Hiersac (16570)' => '16320', + 'Saint-Genis-de-Saintonge (17240)' => '17331', + 'Saint-Genis-du-Bois (33760)' => '33409', + 'Saint-Georges (16700)' => '16321', + 'Saint-Georges (47370)' => '47328', + 'Saint-Georges-Antignac (17240)' => '17332', + 'Saint-Georges-Blancaneix (24130)' => '24413', + 'Saint-Georges-d\'Oléron (17190)' => '17337', + 'Saint-Georges-de-Didonne (17110)' => '17333', + 'Saint-Georges-de-Longuepierre (17470)' => '17334', + 'Saint-Georges-de-Montclard (24140)' => '24414', + 'Saint-Georges-de-Noisné (79400)' => '79253', + 'Saint-Georges-de-Rex (79210)' => '79254', + 'Saint-Georges-des-Agoûts (17150)' => '17335', + 'Saint-Georges-des-Coteaux (17810)' => '17336', + 'Saint-Georges-du-Bois (17700)' => '17338', + 'Saint-Georges-la-Pouge (23250)' => '23197', + 'Saint-Georges-lès-Baillargeaux (86130)' => '86222', + 'Saint-Georges-les-Landes (87160)' => '87145', + 'Saint-Georges-Nigremont (23500)' => '23198', + 'Saint-Geours-d\'Auribat (40380)' => '40260', + 'Saint-Geours-de-Maremne (40230)' => '40261', + 'Saint-Géraud (47120)' => '47245', + 'Saint-Géraud-de-Corps (24700)' => '24415', + 'Saint-Germain (86310)' => '86223', + 'Saint-Germain-Beaupré (23160)' => '23199', + 'Saint-Germain-d\'Esteuil (33340)' => '33412', + 'Saint-Germain-de-Belvès (24170)' => '24416', + 'Saint-Germain-de-Grave (33490)' => '33411', + 'Saint-Germain-de-la-Rivière (33240)' => '33414', + 'Saint-Germain-de-Longue-Chaume (79200)' => '79255', + 'Saint-Germain-de-Lusignan (17500)' => '17339', + 'Saint-Germain-de-Marencennes (17700)' => '17340', + 'Saint-Germain-de-Montbron (16380)' => '16323', + 'Saint-Germain-de-Vibrac (17500)' => '17341', + 'Saint-Germain-des-Prés (24160)' => '24417', + 'Saint-Germain-du-Puch (33750)' => '33413', + 'Saint-Germain-du-Salembre (24190)' => '24418', + 'Saint-Germain-du-Seudre (17240)' => '17342', + 'Saint-Germain-et-Mons (24520)' => '24419', + 'Saint-Germain-Lavolps (19290)' => '19206', + 'Saint-Germain-les-Belles (87380)' => '87146', + 'Saint-Germain-les-Vergnes (19330)' => '19207', + 'Saint-Germier (79340)' => '79256', + 'Saint-Gervais (33240)' => '33415', + 'Saint-Gervais-les-Trois-Clochers (86230)' => '86224', + 'Saint-Géry (24400)' => '24420', + 'Saint-Geyrac (24330)' => '24421', + 'Saint-Gilles-les-Forêts (87130)' => '87147', + 'Saint-Girons-d\'Aiguevives (33920)' => '33416', + 'Saint-Girons-en-Béarn (64300)' => '64479', + 'Saint-Gladie-Arrive-Munein (64390)' => '64480', + 'Saint-Goin (64400)' => '64481', + 'Saint-Gor (40120)' => '40262', + 'Saint-Gourson (16700)' => '16325', + 'Saint-Goussaud (23430)' => '23200', + 'Saint-Grégoire-d\'Ardennes (17240)' => '17343', + 'Saint-Groux (16230)' => '16326', + 'Saint-Hilaire-Bonneval (87260)' => '87148', + 'Saint-Hilaire-d\'Estissac (24140)' => '24422', + 'Saint-Hilaire-de-la-Noaille (33190)' => '33418', + 'Saint-Hilaire-de-Lusignan (47450)' => '47246', + 'Saint-Hilaire-de-Villefranche (17770)' => '17344', + 'Saint-Hilaire-du-Bois (17500)' => '17345', + 'Saint-Hilaire-du-Bois (33540)' => '33419', + 'Saint-Hilaire-Foissac (19550)' => '19208', + 'Saint-Hilaire-la-Palud (79210)' => '79257', + 'Saint-Hilaire-la-Plaine (23150)' => '23201', + 'Saint-Hilaire-la-Treille (87190)' => '87149', + 'Saint-Hilaire-le-Château (23250)' => '23202', + 'Saint-Hilaire-les-Courbes (19170)' => '19209', + 'Saint-Hilaire-les-Places (87800)' => '87150', + 'Saint-Hilaire-Luc (19160)' => '19210', + 'Saint-Hilaire-Peyroux (19560)' => '19211', + 'Saint-Hilaire-Taurieux (19400)' => '19212', + 'Saint-Hippolyte (17430)' => '17346', + 'Saint-Hippolyte (33330)' => '33420', + 'Saint-Jacques-de-Thouars (79100)' => '79258', + 'Saint-Jal (19700)' => '19213', + 'Saint-Jammes (64160)' => '64482', + 'Saint-Jean-d\'Angély (17400)' => '17347', + 'Saint-Jean-d\'Angle (17620)' => '17348', + 'Saint-Jean-d\'Ataux (24190)' => '24424', + 'Saint-Jean-d\'Estissac (24140)' => '24426', + 'Saint-Jean-d\'Eyraud (24140)' => '24427', + 'Saint-Jean-d\'Illac (33127)' => '33422', + 'Saint-Jean-de-Blaignac (33420)' => '33421', + 'Saint-Jean-de-Côle (24800)' => '24425', + 'Saint-Jean-de-Duras (47120)' => '47247', + 'Saint-Jean-de-Lier (40380)' => '40263', + 'Saint-Jean-de-Liversay (17170)' => '17349', + 'Saint-Jean-de-Luz (64500)' => '64483', + 'Saint-Jean-de-Marsacq (40230)' => '40264', + 'Saint-Jean-de-Sauves (86330)' => '86225', + 'Saint-Jean-de-Thouars (79100)' => '79259', + 'Saint-Jean-de-Thurac (47270)' => '47248', + 'Saint-Jean-le-Vieux (64220)' => '64484', + 'Saint-Jean-Ligoure (87260)' => '87151', + 'Saint-Jean-Pied-de-Port (64220)' => '64485', + 'Saint-Jean-Poudge (64330)' => '64486', + 'Saint-Jory-de-Chalais (24800)' => '24428', + 'Saint-Jory-las-Bloux (24160)' => '24429', + 'Saint-Jouin-de-Marnes (79600)' => '79260', + 'Saint-Jouin-de-Milly (79380)' => '79261', + 'Saint-Jouvent (87510)' => '87152', + 'Saint-Julien-aux-Bois (19220)' => '19214', + 'Saint-Julien-Beychevelle (33250)' => '33423', + 'Saint-Julien-d\'Armagnac (40240)' => '40265', + 'Saint-Julien-d\'Eymet (24500)' => '24433', + 'Saint-Julien-de-Crempse (24140)' => '24431', + 'Saint-Julien-de-l\'Escap (17400)' => '17350', + 'Saint-Julien-de-Lampon (24370)' => '24432', + 'Saint-Julien-en-Born (40170)' => '40266', + 'Saint-Julien-l\'Ars (86800)' => '86226', + 'Saint-Julien-la-Genête (23110)' => '23203', + 'Saint-Julien-le-Châtel (23130)' => '23204', + 'Saint-Julien-le-Pèlerin (19430)' => '19215', + 'Saint-Julien-le-Petit (87460)' => '87153', + 'Saint-Julien-le-Vendômois (19210)' => '19216', + 'Saint-Julien-Maumont (19500)' => '19217', + 'Saint-Julien-près-Bort (19110)' => '19218', + 'Saint-Junien (87200)' => '87154', + 'Saint-Junien-la-Bregère (23400)' => '23205', + 'Saint-Junien-les-Combes (87300)' => '87155', + 'Saint-Just (24320)' => '24434', + 'Saint-Just-Ibarre (64120)' => '64487', + 'Saint-Just-le-Martel (87590)' => '87156', + 'Saint-Just-Luzac (17320)' => '17351', + 'Saint-Justin (40240)' => '40267', + 'Saint-Laon (86200)' => '86227', + 'Saint-Laurent (23000)' => '23206', + 'Saint-Laurent (47130)' => '47249', + 'Saint-Laurent-Bretagne (64160)' => '64488', + 'Saint-Laurent-d\'Arce (33240)' => '33425', + 'Saint-Laurent-de-Belzagot (16190)' => '16328', + 'Saint-Laurent-de-Céris (16450)' => '16329', + 'Saint-Laurent-de-Cognac (16100)' => '16330', + 'Saint-Laurent-de-Gosse (40390)' => '40268', + 'Saint-Laurent-de-Jourdes (86410)' => '86228', + 'Saint-Laurent-de-la-Barrière (17380)' => '17352', + 'Saint-Laurent-de-la-Prée (17450)' => '17353', + 'Saint-Laurent-des-Combes (16480)' => '16331', + 'Saint-Laurent-des-Combes (33330)' => '33426', + 'Saint-Laurent-des-Hommes (24400)' => '24436', + 'Saint-Laurent-des-Vignes (24100)' => '24437', + 'Saint-Laurent-du-Bois (33540)' => '33427', + 'Saint-Laurent-du-Plan (33190)' => '33428', + 'Saint-Laurent-la-Vallée (24170)' => '24438', + 'Saint-Laurent-les-Églises (87240)' => '87157', + 'Saint-Laurent-Médoc (33112)' => '33424', + 'Saint-Laurent-sur-Gorre (87310)' => '87158', + 'Saint-Laurs (79160)' => '79263', + 'Saint-Léger (16250)' => '16332', + 'Saint-Léger (17800)' => '17354', + 'Saint-Léger (47160)' => '47250', + 'Saint-Léger-Bridereix (23300)' => '23207', + 'Saint-Léger-de-Balson (33113)' => '33429', + 'Saint-Léger-de-la-Martinière (79500)' => '79264', + 'Saint-Léger-de-Montbrillais (86120)' => '86229', + 'Saint-Léger-de-Montbrun (79100)' => '79265', + 'Saint-Léger-la-Montagne (87340)' => '87159', + 'Saint-Léger-le-Guérétois (23000)' => '23208', + 'Saint-Léger-Magnazeix (87190)' => '87160', + 'Saint-Léomer (86290)' => '86230', + 'Saint-Léon (33670)' => '33431', + 'Saint-Léon (47160)' => '47251', + 'Saint-Léon-d\'Issigeac (24560)' => '24441', + 'Saint-Léon-sur-l\'Isle (24110)' => '24442', + 'Saint-Léon-sur-Vézère (24290)' => '24443', + 'Saint-Léonard-de-Noblat (87400)' => '87161', + 'Saint-Lin (79420)' => '79267', + 'Saint-Lon-les-Mines (40300)' => '40269', + 'Saint-Loubert (33210)' => '33432', + 'Saint-Loubès (33450)' => '33433', + 'Saint-Loubouer (40320)' => '40270', + 'Saint-Louis-de-Montferrand (33440)' => '33434', + 'Saint-Louis-en-l\'Isle (24400)' => '24444', + 'Saint-Loup (17380)' => '17356', + 'Saint-Loup (23130)' => '23209', + 'Saint-Loup-Lamairé (79600)' => '79268', + 'Saint-Macaire (33490)' => '33435', + 'Saint-Macoux (86400)' => '86231', + 'Saint-Magne (33125)' => '33436', + 'Saint-Magne-de-Castillon (33350)' => '33437', + 'Saint-Maigrin (17520)' => '17357', + 'Saint-Maime-de-Péreyrol (24380)' => '24459', + 'Saint-Maixant (23200)' => '23210', + 'Saint-Maixant (33490)' => '33438', + 'Saint-Maixent-de-Beugné (79160)' => '79269', + 'Saint-Maixent-l\'École (79400)' => '79270', + 'Saint-Mandé-sur-Brédoire (17470)' => '17358', + 'Saint-Marc-à-Frongier (23200)' => '23211', + 'Saint-Marc-à-Loubaud (23460)' => '23212', + 'Saint-Marc-la-Lande (79310)' => '79271', + 'Saint-Marcel-du-Périgord (24510)' => '24445', + 'Saint-Marcory (24540)' => '24446', + 'Saint-Mard (17700)' => '17359', + 'Saint-Marien (23600)' => '23213', + 'Saint-Mariens (33620)' => '33439', + 'Saint-Martial (16190)' => '16334', + 'Saint-Martial (17330)' => '17361', + 'Saint-Martial (33490)' => '33440', + 'Saint-Martial-d\'Albarède (24160)' => '24448', + 'Saint-Martial-d\'Artenset (24700)' => '24449', + 'Saint-Martial-de-Gimel (19150)' => '19220', + 'Saint-Martial-de-Mirambeau (17150)' => '17362', + 'Saint-Martial-de-Nabirat (24250)' => '24450', + 'Saint-Martial-de-Valette (24300)' => '24451', + 'Saint-Martial-de-Vitaterne (17500)' => '17363', + 'Saint-Martial-Entraygues (19400)' => '19221', + 'Saint-Martial-le-Mont (23150)' => '23214', + 'Saint-Martial-le-Vieux (23100)' => '23215', + 'Saint-Martial-sur-Isop (87330)' => '87163', + 'Saint-Martial-sur-Né (17520)' => '17364', + 'Saint-Martial-Viveyrol (24320)' => '24452', + 'Saint-Martin-Château (23460)' => '23216', + 'Saint-Martin-Curton (47700)' => '47254', + 'Saint-Martin-d\'Arberoue (64640)' => '64489', + 'Saint-Martin-d\'Arrossa (64780)' => '64490', + 'Saint-Martin-d\'Ary (17270)' => '17365', + 'Saint-Martin-d\'Oney (40090)' => '40274', + 'Saint-Martin-de-Beauville (47270)' => '47255', + 'Saint-Martin-de-Bernegoue (79230)' => '79273', + 'Saint-Martin-de-Coux (17360)' => '17366', + 'Saint-Martin-de-Fressengeas (24800)' => '24453', + 'Saint-Martin-de-Gurson (24610)' => '24454', + 'Saint-Martin-de-Hinx (40390)' => '40272', + 'Saint-Martin-de-Juillers (17400)' => '17367', + 'Saint-Martin-de-Jussac (87200)' => '87164', + 'Saint-Martin-de-Laye (33910)' => '33442', + 'Saint-Martin-de-Lerm (33540)' => '33443', + 'Saint-Martin-de-Mâcon (79100)' => '79274', + 'Saint-Martin-de-Ré (17410)' => '17369', + 'Saint-Martin-de-Ribérac (24600)' => '24455', + 'Saint-Martin-de-Saint-Maixent (79400)' => '79276', + 'Saint-Martin-de-Sanzay (79290)' => '79277', + 'Saint-Martin-de-Seignanx (40390)' => '40273', + 'Saint-Martin-de-Sescas (33490)' => '33444', + 'Saint-Martin-de-Villeréal (47210)' => '47256', + 'Saint-Martin-des-Combes (24140)' => '24456', + 'Saint-Martin-du-Bois (33910)' => '33445', + 'Saint-Martin-du-Clocher (16700)' => '16335', + 'Saint-Martin-du-Fouilloux (79420)' => '79278', + 'Saint-Martin-du-Puy (33540)' => '33446', + 'Saint-Martin-l\'Ars (86350)' => '86234', + 'Saint-Martin-l\'Astier (24400)' => '24457', + 'Saint-Martin-la-Méanne (19320)' => '19222', + 'Saint-Martin-Lacaussade (33390)' => '33441', + 'Saint-Martin-le-Mault (87360)' => '87165', + 'Saint-Martin-le-Pin (24300)' => '24458', + 'Saint-Martin-le-Vieux (87700)' => '87166', + 'Saint-Martin-lès-Melle (79500)' => '79279', + 'Saint-Martin-Petit (47180)' => '47257', + 'Saint-Martin-Sainte-Catherine (23430)' => '23217', + 'Saint-Martin-Sepert (19210)' => '19223', + 'Saint-Martin-Terressus (87400)' => '87167', + 'Saint-Mary (16260)' => '16336', + 'Saint-Mathieu (87440)' => '87168', + 'Saint-Maurice-de-Lestapel (47290)' => '47259', + 'Saint-Maurice-des-Lions (16500)' => '16337', + 'Saint-Maurice-la-Clouère (86160)' => '86235', + 'Saint-Maurice-la-Souterraine (23300)' => '23219', + 'Saint-Maurice-les-Brousses (87800)' => '87169', + 'Saint-Maurice-près-Crocq (23260)' => '23218', + 'Saint-Maurice-sur-Adour (40270)' => '40275', + 'Saint-Maurin (47270)' => '47260', + 'Saint-Maxire (79410)' => '79281', + 'Saint-Méard (87130)' => '87170', + 'Saint-Méard-de-Drône (24600)' => '24460', + 'Saint-Méard-de-Gurçon (24610)' => '24461', + 'Saint-Médard (16300)' => '16338', + 'Saint-Médard (17500)' => '17372', + 'Saint-Médard (64370)' => '64491', + 'Saint-Médard (79370)' => '79282', + 'Saint-Médard-d\'Aunis (17220)' => '17373', + 'Saint-Médard-d\'Excideuil (24160)' => '24463', + 'Saint-Médard-d\'Eyrans (33650)' => '33448', + 'Saint-Médard-de-Guizières (33230)' => '33447', + 'Saint-Médard-de-Mussidan (24400)' => '24462', + 'Saint-Médard-en-Jalles (33160)' => '33449', + 'Saint-Médard-la-Rochette (23200)' => '23220', + 'Saint-Même-les-Carrières (16720)' => '16340', + 'Saint-Merd-de-Lapleau (19320)' => '19225', + 'Saint-Merd-la-Breuille (23100)' => '23221', + 'Saint-Merd-les-Oussines (19170)' => '19226', + 'Saint-Mesmin (24270)' => '24464', + 'Saint-Mexant (19330)' => '19227', + 'Saint-Michel (16470)' => '16341', + 'Saint-Michel (64220)' => '64492', + 'Saint-Michel-de-Castelnau (33840)' => '33450', + 'Saint-Michel-de-Double (24400)' => '24465', + 'Saint-Michel-de-Fronsac (33126)' => '33451', + 'Saint-Michel-de-Lapujade (33190)' => '33453', + 'Saint-Michel-de-Montaigne (24230)' => '24466', + 'Saint-Michel-de-Rieufret (33720)' => '33452', + 'Saint-Michel-de-Veisse (23480)' => '23222', + 'Saint-Michel-de-Villadeix (24380)' => '24468', + 'Saint-Michel-Escalus (40550)' => '40276', + 'Saint-Moreil (23400)' => '23223', + 'Saint-Morillon (33650)' => '33454', + 'Saint-Nazaire-sur-Charente (17780)' => '17375', + 'Saint-Nexans (24520)' => '24472', + 'Saint-Nicolas-de-la-Balerme (47220)' => '47262', + 'Saint-Oradoux-de-Chirouze (23100)' => '23224', + 'Saint-Oradoux-près-Crocq (23260)' => '23225', + 'Saint-Ouen-d\'Aunis (17230)' => '17376', + 'Saint-Ouen-la-Thène (17490)' => '17377', + 'Saint-Ouen-sur-Gartempe (87300)' => '87172', + 'Saint-Palais (33820)' => '33456', + 'Saint-Palais (64120)' => '64493', + 'Saint-Palais-de-Négrignac (17210)' => '17378', + 'Saint-Palais-de-Phiolin (17800)' => '17379', + 'Saint-Palais-du-Né (16300)' => '16342', + 'Saint-Palais-sur-Mer (17420)' => '17380', + 'Saint-Pancrace (24530)' => '24474', + 'Saint-Pandelon (40180)' => '40277', + 'Saint-Pantaléon-de-Lapleau (19160)' => '19228', + 'Saint-Pantaléon-de-Larche (19600)' => '19229', + 'Saint-Pantaly-d\'Ans (24640)' => '24475', + 'Saint-Pantaly-d\'Excideuil (24160)' => '24476', + 'Saint-Pardon-de-Conques (33210)' => '33457', + 'Saint-Pardoult (17400)' => '17381', + 'Saint-Pardoux (79310)' => '79285', + 'Saint-Pardoux (87250)' => '87173', + 'Saint-Pardoux-Corbier (19210)' => '19230', + 'Saint-Pardoux-d\'Arnet (23260)' => '23226', + 'Saint-Pardoux-de-Drône (24600)' => '24477', + 'Saint-Pardoux-du-Breuil (47200)' => '47263', + 'Saint-Pardoux-et-Vielvic (24170)' => '24478', + 'Saint-Pardoux-Isaac (47800)' => '47264', + 'Saint-Pardoux-l\'Ortigier (19270)' => '19234', + 'Saint-Pardoux-la-Croisille (19320)' => '19231', + 'Saint-Pardoux-la-Rivière (24470)' => '24479', + 'Saint-Pardoux-le-Neuf (19200)' => '19232', + 'Saint-Pardoux-le-Neuf (23200)' => '23228', + 'Saint-Pardoux-le-Vieux (19200)' => '19233', + 'Saint-Pardoux-les-Cards (23150)' => '23229', + 'Saint-Pardoux-Morterolles (23400)' => '23227', + 'Saint-Pastour (47290)' => '47265', + 'Saint-Paul (19150)' => '19235', + 'Saint-Paul (33390)' => '33458', + 'Saint-Paul (87260)' => '87174', + 'Saint-Paul-de-Serre (24380)' => '24480', + 'Saint-Paul-en-Born (40200)' => '40278', + 'Saint-Paul-en-Gâtine (79240)' => '79286', + 'Saint-Paul-la-Roche (24800)' => '24481', + 'Saint-Paul-lès-Dax (40990)' => '40279', + 'Saint-Paul-Lizonne (24320)' => '24482', + 'Saint-Pé-de-Léren (64270)' => '64494', + 'Saint-Pé-Saint-Simon (47170)' => '47266', + 'Saint-Pée-sur-Nivelle (64310)' => '64495', + 'Saint-Perdon (40090)' => '40280', + 'Saint-Perdoux (24560)' => '24483', + 'Saint-Pey-d\'Armens (33330)' => '33459', + 'Saint-Pey-de-Castets (33350)' => '33460', + 'Saint-Philippe-d\'Aiguille (33350)' => '33461', + 'Saint-Philippe-du-Seignal (33220)' => '33462', + 'Saint-Pierre-Bellevue (23460)' => '23232', + 'Saint-Pierre-Chérignat (23430)' => '23230', + 'Saint-Pierre-d\'Amilly (17700)' => '17382', + 'Saint-Pierre-d\'Aurillac (33490)' => '33463', + 'Saint-Pierre-d\'Exideuil (86400)' => '86237', + 'Saint-Pierre-d\'Eyraud (24130)' => '24487', + 'Saint-Pierre-d\'Irube (64990)' => '64496', + 'Saint-Pierre-d\'Oléron (17310)' => '17385', + 'Saint-Pierre-de-Bat (33760)' => '33464', + 'Saint-Pierre-de-Buzet (47160)' => '47267', + 'Saint-Pierre-de-Chignac (24330)' => '24484', + 'Saint-Pierre-de-Clairac (47270)' => '47269', + 'Saint-Pierre-de-Côle (24800)' => '24485', + 'Saint-Pierre-de-Frugie (24450)' => '24486', + 'Saint-Pierre-de-Fursac (23290)' => '23231', + 'Saint-Pierre-de-Juillers (17400)' => '17383', + 'Saint-Pierre-de-l\'Isle (17330)' => '17384', + 'Saint-Pierre-de-Maillé (86260)' => '86236', + 'Saint-Pierre-de-Mons (33210)' => '33465', + 'Saint-Pierre-des-Échaubrognes (79700)' => '79289', + 'Saint-Pierre-du-Mont (40280)' => '40281', + 'Saint-Pierre-du-Palais (17270)' => '17386', + 'Saint-Pierre-le-Bost (23600)' => '23233', + 'Saint-Pierre-sur-Dropt (47120)' => '47271', + 'Saint-Pompain (79160)' => '79290', + 'Saint-Pompont (24170)' => '24488', + 'Saint-Porchaire (17250)' => '17387', + 'Saint-Preuil (16130)' => '16343', + 'Saint-Priest (23110)' => '23234', + 'Saint-Priest-de-Gimel (19800)' => '19236', + 'Saint-Priest-la-Feuille (23300)' => '23235', + 'Saint-Priest-la-Plaine (23240)' => '23236', + 'Saint-Priest-les-Fougères (24450)' => '24489', + 'Saint-Priest-Ligoure (87800)' => '87176', + 'Saint-Priest-Palus (23400)' => '23237', + 'Saint-Priest-sous-Aixe (87700)' => '87177', + 'Saint-Priest-Taurion (87480)' => '87178', + 'Saint-Privat (19220)' => '19237', + 'Saint-Privat-des-Prés (24410)' => '24490', + 'Saint-Projet-Saint-Constant (16110)' => '16344', + 'Saint-Quantin-de-Rançanne (17800)' => '17388', + 'Saint-Quentin-de-Baron (33750)' => '33466', + 'Saint-Quentin-de-Caplong (33220)' => '33467', + 'Saint-Quentin-de-Chalais (16210)' => '16346', + 'Saint-Quentin-du-Dropt (47330)' => '47272', + 'Saint-Quentin-la-Chabanne (23500)' => '23238', + 'Saint-Quentin-sur-Charente (16150)' => '16345', + 'Saint-Rabier (24210)' => '24491', + 'Saint-Raphaël (24160)' => '24493', + 'Saint-Rémy (19290)' => '19238', + 'Saint-Rémy (24700)' => '24494', + 'Saint-Rémy (79410)' => '79293', + 'Saint-Rémy-sur-Creuse (86220)' => '86241', + 'Saint-Robert (19310)' => '19239', + 'Saint-Robert (47340)' => '47273', + 'Saint-Rogatien (17220)' => '17391', + 'Saint-Romain (16210)' => '16347', + 'Saint-Romain (86250)' => '86242', + 'Saint-Romain-de-Benet (17600)' => '17393', + 'Saint-Romain-de-Monpazier (24540)' => '24495', + 'Saint-Romain-et-Saint-Clément (24800)' => '24496', + 'Saint-Romain-la-Virvée (33240)' => '33470', + 'Saint-Romain-le-Noble (47270)' => '47274', + 'Saint-Romain-sur-Gironde (17240)' => '17392', + 'Saint-Romans-des-Champs (79230)' => '79294', + 'Saint-Romans-lès-Melle (79500)' => '79295', + 'Saint-Salvadour (19700)' => '19240', + 'Saint-Salvy (47360)' => '47275', + 'Saint-Sardos (47360)' => '47276', + 'Saint-Saturnin (16290)' => '16348', + 'Saint-Saturnin-du-Bois (17700)' => '17394', + 'Saint-Saud-Lacoussière (24470)' => '24498', + 'Saint-Sauvant (17610)' => '17395', + 'Saint-Sauvant (86600)' => '86244', + 'Saint-Sauveur (24520)' => '24499', + 'Saint-Sauveur (33250)' => '33471', + 'Saint-Sauveur-d\'Aunis (17540)' => '17396', + 'Saint-Sauveur-de-Meilhan (47180)' => '47277', + 'Saint-Sauveur-de-Puynormand (33660)' => '33472', + 'Saint-Sauveur-Lalande (24700)' => '24500', + 'Saint-Savin (33920)' => '33473', + 'Saint-Savin (86310)' => '86246', + 'Saint-Savinien (17350)' => '17397', + 'Saint-Saviol (86400)' => '86247', + 'Saint-Sébastien (23160)' => '23239', + 'Saint-Secondin (86350)' => '86248', + 'Saint-Selve (33650)' => '33474', + 'Saint-Sernin (47120)' => '47278', + 'Saint-Setiers (19290)' => '19241', + 'Saint-Seurin-de-Bourg (33710)' => '33475', + 'Saint-Seurin-de-Cadourne (33180)' => '33476', + 'Saint-Seurin-de-Cursac (33390)' => '33477', + 'Saint-Seurin-de-Palenne (17800)' => '17398', + 'Saint-Seurin-de-Prats (24230)' => '24501', + 'Saint-Seurin-sur-l\'Isle (33660)' => '33478', + 'Saint-Sève (33190)' => '33479', + 'Saint-Sever (40500)' => '40282', + 'Saint-Sever-de-Saintonge (17800)' => '17400', + 'Saint-Séverin (16390)' => '16350', + 'Saint-Séverin-d\'Estissac (24190)' => '24502', + 'Saint-Séverin-sur-Boutonne (17330)' => '17401', + 'Saint-Sigismond-de-Clermont (17240)' => '17402', + 'Saint-Silvain-Bas-le-Roc (23600)' => '23240', + 'Saint-Silvain-Bellegarde (23190)' => '23241', + 'Saint-Silvain-Montaigut (23320)' => '23242', + 'Saint-Silvain-sous-Toulx (23140)' => '23243', + 'Saint-Simeux (16120)' => '16351', + 'Saint-Simon (16120)' => '16352', + 'Saint-Simon-de-Bordes (17500)' => '17403', + 'Saint-Simon-de-Pellouaille (17260)' => '17404', + 'Saint-Sixte (47220)' => '47279', + 'Saint-Solve (19130)' => '19242', + 'Saint-Sorlin-de-Conac (17150)' => '17405', + 'Saint-Sornin (16220)' => '16353', + 'Saint-Sornin (17600)' => '17406', + 'Saint-Sornin-la-Marche (87210)' => '87179', + 'Saint-Sornin-Lavolps (19230)' => '19243', + 'Saint-Sornin-Leulac (87290)' => '87180', + 'Saint-Sulpice-d\'Arnoult (17250)' => '17408', + 'Saint-Sulpice-d\'Excideuil (24800)' => '24505', + 'Saint-Sulpice-de-Cognac (16370)' => '16355', + 'Saint-Sulpice-de-Faleyrens (33330)' => '33480', + 'Saint-Sulpice-de-Guilleragues (33580)' => '33481', + 'Saint-Sulpice-de-Mareuil (24340)' => '24503', + 'Saint-Sulpice-de-Pommiers (33540)' => '33482', + 'Saint-Sulpice-de-Roumagnac (24600)' => '24504', + 'Saint-Sulpice-de-Royan (17200)' => '17409', + 'Saint-Sulpice-de-Ruffec (16460)' => '16356', + 'Saint-Sulpice-et-Cameyrac (33450)' => '33483', + 'Saint-Sulpice-Laurière (87370)' => '87181', + 'Saint-Sulpice-le-Dunois (23800)' => '23244', + 'Saint-Sulpice-le-Guérétois (23000)' => '23245', + 'Saint-Sulpice-les-Bois (19250)' => '19244', + 'Saint-Sulpice-les-Champs (23480)' => '23246', + 'Saint-Sulpice-les-Feuilles (87160)' => '87182', + 'Saint-Sylvain (19380)' => '19245', + 'Saint-Sylvestre (87240)' => '87183', + 'Saint-Sylvestre-sur-Lot (47140)' => '47280', + 'Saint-Symphorien (33113)' => '33484', + 'Saint-Symphorien (79270)' => '79298', + 'Saint-Symphorien-sur-Couze (87140)' => '87184', + 'Saint-Thomas-de-Conac (17150)' => '17410', + 'Saint-Trojan (33710)' => '33486', + 'Saint-Trojan-les-Bains (17370)' => '17411', + 'Saint-Urcisse (47270)' => '47281', + 'Saint-Vaize (17100)' => '17412', + 'Saint-Vallier (16480)' => '16357', + 'Saint-Varent (79330)' => '79299', + 'Saint-Vaury (23320)' => '23247', + 'Saint-Viance (19240)' => '19246', + 'Saint-Victor (24350)' => '24508', + 'Saint-Victor-en-Marche (23000)' => '23248', + 'Saint-Victour (19200)' => '19247', + 'Saint-Victurnien (87420)' => '87185', + 'Saint-Vincent (64800)' => '64498', + 'Saint-Vincent-de-Connezac (24190)' => '24509', + 'Saint-Vincent-de-Cosse (24220)' => '24510', + 'Saint-Vincent-de-Lamontjoie (47310)' => '47282', + 'Saint-Vincent-de-Paul (33440)' => '33487', + 'Saint-Vincent-de-Paul (40990)' => '40283', + 'Saint-Vincent-de-Pertignas (33420)' => '33488', + 'Saint-Vincent-de-Tyrosse (40230)' => '40284', + 'Saint-Vincent-Jalmoutiers (24410)' => '24511', + 'Saint-Vincent-la-Châtre (79500)' => '79301', + 'Saint-Vincent-le-Paluel (24200)' => '24512', + 'Saint-Vincent-sur-l\'Isle (24420)' => '24513', + 'Saint-Vite (47500)' => '47283', + 'Saint-Vitte-sur-Briance (87380)' => '87186', + 'Saint-Vivien (17220)' => '17413', + 'Saint-Vivien (24230)' => '24514', + 'Saint-Vivien-de-Blaye (33920)' => '33489', + 'Saint-Vivien-de-Médoc (33590)' => '33490', + 'Saint-Vivien-de-Monségur (33580)' => '33491', + 'Saint-Xandre (17138)' => '17414', + 'Saint-Yaguen (40400)' => '40285', + 'Saint-Ybard (19140)' => '19248', + 'Saint-Yrieix-la-Montagne (23460)' => '23249', + 'Saint-Yrieix-la-Perche (87500)' => '87187', + 'Saint-Yrieix-le-Déjalat (19300)' => '19249', + 'Saint-Yrieix-les-Bois (23150)' => '23250', + 'Saint-Yrieix-sous-Aixe (87700)' => '87188', + 'Saint-Yrieix-sur-Charente (16710)' => '16358', + 'Saint-Yzan-de-Soudiac (33920)' => '33492', + 'Saint-Yzans-de-Médoc (33340)' => '33493', + 'Sainte-Alvère-Saint-Laurent Les Bâtons (24510)' => '24362', + 'Sainte-Anne-Saint-Priest (87120)' => '87134', + 'Sainte-Bazeille (47180)' => '47233', + 'Sainte-Blandine (79370)' => '79240', + 'Sainte-Colombe (16230)' => '16309', + 'Sainte-Colombe (17210)' => '17319', + 'Sainte-Colombe (33350)' => '33390', + 'Sainte-Colombe (40700)' => '40252', + 'Sainte-Colombe-de-Duras (47120)' => '47236', + 'Sainte-Colombe-de-Villeneuve (47300)' => '47237', + 'Sainte-Colombe-en-Bruilhois (47310)' => '47238', + 'Sainte-Colome (64260)' => '64473', + 'Sainte-Croix (24440)' => '24393', + 'Sainte-Croix-de-Mareuil (24340)' => '24394', + 'Sainte-Croix-du-Mont (33410)' => '33392', + 'Sainte-Eanne (79800)' => '79246', + 'Sainte-Engrâce (64560)' => '64475', + 'Sainte-Eulalie (33560)' => '33397', + 'Sainte-Eulalie-d\'Ans (24640)' => '24401', + 'Sainte-Eulalie-d\'Eymet (24500)' => '24402', + 'Sainte-Eulalie-en-Born (40200)' => '40257', + 'Sainte-Féréole (19270)' => '19202', + 'Sainte-Feyre (23000)' => '23193', + 'Sainte-Feyre-la-Montagne (23500)' => '23194', + 'Sainte-Florence (33350)' => '33401', + 'Sainte-Fortunade (19490)' => '19203', + 'Sainte-Foy (40190)' => '40258', + 'Sainte-Foy-de-Belvès (24170)' => '24406', + 'Sainte-Foy-de-Longas (24510)' => '24407', + 'Sainte-Foy-la-Grande (33220)' => '33402', + 'Sainte-Foy-la-Longue (33490)' => '33403', + 'Sainte-Gemme (17250)' => '17330', + 'Sainte-Gemme (33580)' => '33404', + 'Sainte-Gemme (79330)' => '79250', + 'Sainte-Gemme-Martaillac (47250)' => '47244', + 'Sainte-Hélène (33480)' => '33417', + 'Sainte-Innocence (24500)' => '24423', + 'Sainte-Lheurine (17520)' => '17355', + 'Sainte-Livrade-sur-Lot (47110)' => '47252', + 'Sainte-Marie-de-Chignac (24330)' => '24447', + 'Sainte-Marie-de-Gosse (40390)' => '40271', + 'Sainte-Marie-de-Ré (17740)' => '17360', + 'Sainte-Marie-de-Vaux (87420)' => '87162', + 'Sainte-Marie-Lapanouze (19160)' => '19219', + 'Sainte-Marthe (47430)' => '47253', + 'Sainte-Maure-de-Peyriac (47170)' => '47258', + 'Sainte-Même (17770)' => '17374', + 'Sainte-Mondane (24370)' => '24470', + 'Sainte-Nathalène (24200)' => '24471', + 'Sainte-Néomaye (79260)' => '79283', + 'Sainte-Orse (24210)' => '24473', + 'Sainte-Ouenne (79220)' => '79284', + 'Sainte-Radegonde (17250)' => '17389', + 'Sainte-Radegonde (24560)' => '24492', + 'Sainte-Radegonde (33350)' => '33468', + 'Sainte-Radegonde (79100)' => '79292', + 'Sainte-Radégonde (86300)' => '86239', + 'Sainte-Ramée (17240)' => '17390', + 'Sainte-Sévère (16200)' => '16349', + 'Sainte-Soline (79120)' => '79297', + 'Sainte-Souline (16480)' => '16354', + 'Sainte-Soulle (17220)' => '17407', + 'Sainte-Terre (33350)' => '33485', + 'Sainte-Trie (24160)' => '24507', + 'Sainte-Verge (79100)' => '79300', + 'Saintes (17100)' => '17415', + 'Saires (86420)' => '86249', + 'Saivres (79400)' => '79302', + 'Saix (86120)' => '86250', + 'Salagnac (24160)' => '24515', + 'Salaunes (33160)' => '33494', + 'Saleignes (17510)' => '17416', + 'Salies-de-Béarn (64270)' => '64499', + 'Salignac-de-Mirambeau (17130)' => '17417', + 'Salignac-Eyvigues (24590)' => '24516', + 'Salignac-sur-Charente (17800)' => '17418', + 'Salleboeuf (33370)' => '33496', + 'Salles (33770)' => '33498', + 'Salles (47150)' => '47284', + 'Salles (79800)' => '79303', + 'Salles-d\'Angles (16130)' => '16359', + 'Salles-de-Barbezieux (16300)' => '16360', + 'Salles-de-Belvès (24170)' => '24517', + 'Salles-de-Villefagnan (16700)' => '16361', + 'Salles-Lavalette (16190)' => '16362', + 'Salles-Mongiscard (64300)' => '64500', + 'Salles-sur-Mer (17220)' => '17420', + 'Sallespisse (64300)' => '64501', + 'Salon (24380)' => '24518', + 'Salon-la-Tour (19510)' => '19250', + 'Samadet (40320)' => '40286', + 'Samazan (47250)' => '47285', + 'Sames (64520)' => '64502', + 'Sammarçolles (86200)' => '86252', + 'Samonac (33710)' => '33500', + 'Samsons-Lion (64350)' => '64503', + 'Sanguinet (40460)' => '40287', + 'Sannat (23110)' => '23167', + 'Sansais (79270)' => '79304', + 'Sanxay (86600)' => '86253', + 'Sarbazan (40120)' => '40288', + 'Sardent (23250)' => '23168', + 'Sare (64310)' => '64504', + 'Sarlande (24270)' => '24519', + 'Sarlat-la-Canéda (24200)' => '24520', + 'Sarliac-sur-l\'Isle (24420)' => '24521', + 'Sarpourenx (64300)' => '64505', + 'Sarran (19800)' => '19251', + 'Sarrance (64490)' => '64506', + 'Sarrazac (24800)' => '24522', + 'Sarraziet (40500)' => '40289', + 'Sarron (40800)' => '40290', + 'Sarroux (19110)' => '19252', + 'Saubion (40230)' => '40291', + 'Saubole (64420)' => '64507', + 'Saubrigues (40230)' => '40292', + 'Saubusse (40180)' => '40293', + 'Saucats (33650)' => '33501', + 'Saucède (64400)' => '64508', + 'Saugnac-et-Cambran (40180)' => '40294', + 'Saugnacq-et-Muret (40410)' => '40295', + 'Saugon (33920)' => '33502', + 'Sauguis-Saint-Étienne (64470)' => '64509', + 'Saujon (17600)' => '17421', + 'Saulgé (86500)' => '86254', + 'Saulgond (16420)' => '16363', + 'Sault-de-Navailles (64300)' => '64510', + 'Sauméjan (47420)' => '47286', + 'Saumont (47600)' => '47287', + 'Saumos (33680)' => '33503', + 'Saurais (79200)' => '79306', + 'Saussignac (24240)' => '24523', + 'Sauternes (33210)' => '33504', + 'Sauvagnac (16310)' => '16364', + 'Sauvagnas (47340)' => '47288', + 'Sauvagnon (64230)' => '64511', + 'Sauvelade (64150)' => '64512', + 'Sauveterre-de-Béarn (64390)' => '64513', + 'Sauveterre-de-Guyenne (33540)' => '33506', + 'Sauveterre-la-Lémance (47500)' => '47292', + 'Sauveterre-Saint-Denis (47220)' => '47293', + 'Sauviac (33430)' => '33507', + 'Sauviat-sur-Vige (87400)' => '87190', + 'Sauvignac (16480)' => '16365', + 'Sauzé-Vaussais (79190)' => '79307', + 'Savennes (23000)' => '23170', + 'Savignac (33124)' => '33508', + 'Savignac-de-Duras (47120)' => '47294', + 'Savignac-de-l\'Isle (33910)' => '33509', + 'Savignac-de-Miremont (24260)' => '24524', + 'Savignac-de-Nontron (24300)' => '24525', + 'Savignac-Lédrier (24270)' => '24526', + 'Savignac-les-Églises (24420)' => '24527', + 'Savignac-sur-Leyze (47150)' => '47295', + 'Savigné (86400)' => '86255', + 'Savigny-Lévescault (86800)' => '86256', + 'Savigny-sous-Faye (86140)' => '86257', + 'Sceau-Saint-Angel (24300)' => '24528', + 'Sciecq (79000)' => '79308', + 'Scillé (79240)' => '79309', + 'Scorbé-Clairvaux (86140)' => '86258', + 'Séby (64410)' => '64514', + 'Secondigné-sur-Belle (79170)' => '79310', + 'Secondigny (79130)' => '79311', + 'Sedze-Maubecq (64160)' => '64515', + 'Sedzère (64160)' => '64516', + 'Ségalas (47410)' => '47296', + 'Segonzac (16130)' => '16366', + 'Segonzac (19310)' => '19253', + 'Segonzac (24600)' => '24529', + 'Ségur-le-Château (19230)' => '19254', + 'Seigné (17510)' => '17422', + 'Seignosse (40510)' => '40296', + 'Seilhac (19700)' => '19255', + 'Séligné (79170)' => '79312', + 'Sembas (47360)' => '47297', + 'Séméacq-Blachon (64350)' => '64517', + 'Semens (33490)' => '33510', + 'Semillac (17150)' => '17423', + 'Semoussac (17150)' => '17424', + 'Semussac (17120)' => '17425', + 'Sencenac-Puy-de-Fourches (24310)' => '24530', + 'Sendets (33690)' => '33511', + 'Sendets (64320)' => '64518', + 'Sénestis (47430)' => '47298', + 'Senillé-Saint-Sauveur (86100)' => '86245', + 'Sepvret (79120)' => '79313', + 'Sérandon (19160)' => '19256', + 'Séreilhac (87620)' => '87191', + 'Sergeac (24290)' => '24531', + 'Sérignac-Péboudou (47410)' => '47299', + 'Sérignac-sur-Garonne (47310)' => '47300', + 'Sérigny (86230)' => '86260', + 'Sérilhac (19190)' => '19257', + 'Sermur (23700)' => '23171', + 'Séron (65320)' => '65422', + 'Serres-Castet (64121)' => '64519', + 'Serres-et-Montguyard (24500)' => '24532', + 'Serres-Gaston (40700)' => '40298', + 'Serres-Morlaàs (64160)' => '64520', + 'Serres-Sainte-Marie (64170)' => '64521', + 'Serreslous-et-Arribans (40700)' => '40299', + 'Sers (16410)' => '16368', + 'Servanches (24410)' => '24533', + 'Servières-le-Château (19220)' => '19258', + 'Sévignacq (64160)' => '64523', + 'Sévignacq-Meyracq (64260)' => '64522', + 'Sèvres-Anxaumont (86800)' => '86261', + 'Sexcles (19430)' => '19259', + 'Seyches (47350)' => '47301', + 'Seyresse (40180)' => '40300', + 'Siecq (17490)' => '17427', + 'Siest (40180)' => '40301', + 'Sigalens (33690)' => '33512', + 'Sigogne (16200)' => '16369', + 'Sigoulès (24240)' => '24534', + 'Sillars (86320)' => '86262', + 'Sillas (33690)' => '33513', + 'Simacourbe (64350)' => '64524', + 'Simeyrols (24370)' => '24535', + 'Sindères (40110)' => '40302', + 'Singleyrac (24500)' => '24536', + 'Sioniac (19120)' => '19260', + 'Siorac-de-Ribérac (24600)' => '24537', + 'Siorac-en-Périgord (24170)' => '24538', + 'Sireuil (16440)' => '16370', + 'Siros (64230)' => '64525', + 'Smarves (86240)' => '86263', + 'Solférino (40210)' => '40303', + 'Solignac (87110)' => '87192', + 'Sommières-du-Clain (86160)' => '86264', + 'Sompt (79110)' => '79314', + 'Sonnac (17160)' => '17428', + 'Soorts-Hossegor (40150)' => '40304', + 'Sorbets (40320)' => '40305', + 'Sorde-l\'Abbaye (40300)' => '40306', + 'Sore (40430)' => '40307', + 'Sorges et Ligueux en Périgord (24420)' => '24540', + 'Sornac (19290)' => '19261', + 'Sort-en-Chalosse (40180)' => '40308', + 'Sos (47170)' => '47302', + 'Sossais (86230)' => '86265', + 'Soubise (17780)' => '17429', + 'Soubran (17150)' => '17430', + 'Soubrebost (23250)' => '23173', + 'Soudaine-Lavinadière (19370)' => '19262', + 'Soudan (79800)' => '79316', + 'Soudat (24360)' => '24541', + 'Soudeilles (19300)' => '19263', + 'Souffrignac (16380)' => '16372', + 'Soulac-sur-Mer (33780)' => '33514', + 'Soulaures (24540)' => '24542', + 'Soulignac (33760)' => '33515', + 'Soulignonne (17250)' => '17431', + 'Soumans (23600)' => '23174', + 'Soumensac (47120)' => '47303', + 'Souméras (17130)' => '17432', + 'Soumoulou (64420)' => '64526', + 'Souprosse (40250)' => '40309', + 'Souraïde (64250)' => '64527', + 'Soursac (19550)' => '19264', + 'Sourzac (24400)' => '24543', + 'Sous-Parsat (23150)' => '23175', + 'Sousmoulins (17130)' => '17433', + 'Soussac (33790)' => '33516', + 'Soussans (33460)' => '33517', + 'Soustons (40140)' => '40310', + 'Soutiers (79310)' => '79318', + 'Souvigné (16240)' => '16373', + 'Souvigné (79800)' => '79319', + 'Soyaux (16800)' => '16374', + 'Suaux (16260)' => '16375', + 'Suhescun (64780)' => '64528', + 'Surdoux (87130)' => '87193', + 'Surgères (17700)' => '17434', + 'Surin (79220)' => '79320', + 'Surin (86250)' => '86266', + 'Suris (16270)' => '16376', + 'Sus (64190)' => '64529', + 'Susmiou (64190)' => '64530', + 'Sussac (87130)' => '87194', + 'Tabaille-Usquain (64190)' => '64531', + 'Tabanac (33550)' => '33518', + 'Tadousse-Ussau (64330)' => '64532', + 'Taillant (17350)' => '17435', + 'Taillebourg (17350)' => '17436', + 'Taillebourg (47200)' => '47304', + 'Taillecavat (33580)' => '33520', + 'Taizé (79100)' => '79321', + 'Taizé-Aizie (16700)' => '16378', + 'Talais (33590)' => '33521', + 'Talence (33400)' => '33522', + 'Taller (40260)' => '40311', + 'Talmont-sur-Gironde (17120)' => '17437', + 'Tamniès (24620)' => '24544', + 'Tanzac (17260)' => '17438', + 'Taponnat-Fleurignac (16110)' => '16379', + 'Tardes (23170)' => '23251', + 'Tardets-Sorholus (64470)' => '64533', + 'Targon (33760)' => '33523', + 'Tarnac (19170)' => '19265', + 'Tarnès (33240)' => '33524', + 'Tarnos (40220)' => '40312', + 'Taron-Sadirac-Viellenave (64330)' => '64534', + 'Tarsacq (64360)' => '64535', + 'Tartas (40400)' => '40313', + 'Taugon (17170)' => '17439', + 'Tauriac (33710)' => '33525', + 'Tayac (33570)' => '33526', + 'Tayrac (47270)' => '47305', + 'Teillots (24390)' => '24545', + 'Temple-Laguyon (24390)' => '24546', + 'Tercé (86800)' => '86268', + 'Tercillat (23350)' => '23252', + 'Tercis-les-Bains (40180)' => '40314', + 'Ternant (17400)' => '17440', + 'Ternay (86120)' => '86269', + 'Terrasson-Lavilledieu (24120)' => '24547', + 'Tersannes (87360)' => '87195', + 'Tesson (17460)' => '17441', + 'Tessonnière (79600)' => '79325', + 'Téthieu (40990)' => '40315', + 'Teuillac (33710)' => '33530', + 'Teyjat (24300)' => '24548', + 'Thaims (17120)' => '17442', + 'Thairé (17290)' => '17443', + 'Thalamy (19200)' => '19266', + 'Thauron (23250)' => '23253', + 'Theil-Rabier (16240)' => '16381', + 'Thénac (17460)' => '17444', + 'Thénac (24240)' => '24549', + 'Thénezay (79390)' => '79326', + 'Thenon (24210)' => '24550', + 'Thézac (17600)' => '17445', + 'Thézac (47370)' => '47307', + 'Thèze (64450)' => '64536', + 'Thiat (87320)' => '87196', + 'Thiviers (24800)' => '24551', + 'Thollet (86290)' => '86270', + 'Thonac (24290)' => '24552', + 'Thorigné (79370)' => '79327', + 'Thorigny-sur-le-Mignon (79360)' => '79328', + 'Thors (17160)' => '17446', + 'Thouars (79100)' => '79329', + 'Thouars-sur-Garonne (47230)' => '47308', + 'Thouron (87140)' => '87197', + 'Thurageau (86110)' => '86271', + 'Thuré (86540)' => '86272', + 'Tilh (40360)' => '40316', + 'Tillou (79110)' => '79330', + 'Tizac-de-Curton (33420)' => '33531', + 'Tizac-de-Lapouyade (33620)' => '33532', + 'Tocane-Saint-Apre (24350)' => '24553', + 'Tombeboeuf (47380)' => '47309', + 'Tonnay-Boutonne (17380)' => '17448', + 'Tonnay-Charente (17430)' => '17449', + 'Tonneins (47400)' => '47310', + 'Torsac (16410)' => '16382', + 'Torxé (17380)' => '17450', + 'Tosse (40230)' => '40317', + 'Toulenne (33210)' => '33533', + 'Toulouzette (40250)' => '40318', + 'Toulx-Sainte-Croix (23600)' => '23254', + 'Tourliac (47210)' => '47311', + 'Tournon-d\'Agenais (47370)' => '47312', + 'Tourriers (16560)' => '16383', + 'Tourtenay (79100)' => '79331', + 'Tourtoirac (24390)' => '24555', + 'Tourtrès (47380)' => '47313', + 'Touvérac (16360)' => '16384', + 'Touvre (16600)' => '16385', + 'Touzac (16120)' => '16386', + 'Toy-Viam (19170)' => '19268', + 'Trayes (79240)' => '79332', + 'Treignac (19260)' => '19269', + 'Trélissac (24750)' => '24557', + 'Trémolat (24510)' => '24558', + 'Trémons (47140)' => '47314', + 'Trensacq (40630)' => '40319', + 'Trentels (47140)' => '47315', + 'Tresses (33370)' => '33535', + 'Triac-Lautrait (16200)' => '16387', + 'Trizay (17250)' => '17453', + 'Troche (19230)' => '19270', + 'Trois-Fonds (23230)' => '23255', + 'Trois-Palis (16730)' => '16388', + 'Trois-Villes (64470)' => '64537', + 'Tudeils (19120)' => '19271', + 'Tugéras-Saint-Maurice (17130)' => '17454', + 'Tulle (19000)' => '19272', + 'Turenne (19500)' => '19273', + 'Turgon (16350)' => '16389', + 'Tursac (24620)' => '24559', + 'Tusson (16140)' => '16390', + 'Tuzie (16700)' => '16391', + 'Uchacq-et-Parentis (40090)' => '40320', + 'Uhart-Cize (64220)' => '64538', + 'Uhart-Mixe (64120)' => '64539', + 'Urcuit (64990)' => '64540', + 'Urdès (64370)' => '64541', + 'Urdos (64490)' => '64542', + 'Urepel (64430)' => '64543', + 'Urgons (40320)' => '40321', + 'Urost (64160)' => '64544', + 'Urrugne (64122)' => '64545', + 'Urt (64240)' => '64546', + 'Urval (24480)' => '24560', + 'Ussac (19270)' => '19274', + 'Usseau (79210)' => '79334', + 'Usseau (86230)' => '86275', + 'Ussel (19200)' => '19275', + 'Usson-du-Poitou (86350)' => '86276', + 'Ustaritz (64480)' => '64547', + 'Uza (40170)' => '40322', + 'Uzan (64370)' => '64548', + 'Uzein (64230)' => '64549', + 'Uzerche (19140)' => '19276', + 'Uzeste (33730)' => '33537', + 'Uzos (64110)' => '64550', + 'Val d\'Issoire (87330)' => '87097', + 'Val de Virvée (33240)' => '33018', + 'Val des Vignes (16250)' => '16175', + 'Valdivienne (86300)' => '86233', + 'Valence (16460)' => '16392', + 'Valeuil (24310)' => '24561', + 'Valeyrac (33340)' => '33538', + 'Valiergues (19200)' => '19277', + 'Vallans (79270)' => '79335', + 'Vallereuil (24190)' => '24562', + 'Vallière (23120)' => '23257', + 'Valojoulx (24290)' => '24563', + 'Vançais (79120)' => '79336', + 'Vandré (17700)' => '17457', + 'Vanxains (24600)' => '24564', + 'Vanzac (17500)' => '17458', + 'Vanzay (79120)' => '79338', + 'Varaignes (24360)' => '24565', + 'Varaize (17400)' => '17459', + 'Vareilles (23300)' => '23258', + 'Varennes (24150)' => '24566', + 'Varennes (86110)' => '86277', + 'Varès (47400)' => '47316', + 'Varetz (19240)' => '19278', + 'Vars (16330)' => '16393', + 'Vars-sur-Roseix (19130)' => '19279', + 'Varzay (17460)' => '17460', + 'Vasles (79340)' => '79339', + 'Vaulry (87140)' => '87198', + 'Vaunac (24800)' => '24567', + 'Vausseroux (79420)' => '79340', + 'Vautebis (79420)' => '79341', + 'Vaux (86700)' => '86278', + 'Vaux-Lavalette (16320)' => '16394', + 'Vaux-Rouillac (16170)' => '16395', + 'Vaux-sur-Mer (17640)' => '17461', + 'Vaux-sur-Vienne (86220)' => '86279', + 'Vayres (33870)' => '33539', + 'Vayres (87600)' => '87199', + 'Végennes (19120)' => '19280', + 'Veix (19260)' => '19281', + 'Vélines (24230)' => '24568', + 'Vellèches (86230)' => '86280', + 'Vendays-Montalivet (33930)' => '33540', + 'Vendeuvre-du-Poitou (86380)' => '86281', + 'Vendoire (24320)' => '24569', + 'Vénérand (17100)' => '17462', + 'Vensac (33590)' => '33541', + 'Ventouse (16460)' => '16396', + 'Vérac (33240)' => '33542', + 'Verdelais (33490)' => '33543', + 'Verdets (64400)' => '64551', + 'Verdille (16140)' => '16397', + 'Verdon (24520)' => '24570', + 'Vergeroux (17300)' => '17463', + 'Vergné (17330)' => '17464', + 'Vergt (24380)' => '24571', + 'Vergt-de-Biron (24540)' => '24572', + 'Vérines (17540)' => '17466', + 'Verneiges (23170)' => '23259', + 'Verneuil (16310)' => '16398', + 'Verneuil-Moustiers (87360)' => '87200', + 'Verneuil-sur-Vienne (87430)' => '87201', + 'Vernon (86340)' => '86284', + 'Vernoux-en-Gâtine (79240)' => '79342', + 'Vernoux-sur-Boutonne (79170)' => '79343', + 'Verrières (16130)' => '16399', + 'Verrières (86410)' => '86285', + 'Verrue (86420)' => '86286', + 'Verruyes (79310)' => '79345', + 'Vert (40420)' => '40323', + 'Verteillac (24320)' => '24573', + 'Verteuil-d\'Agenais (47260)' => '47317', + 'Verteuil-sur-Charente (16510)' => '16400', + 'Vertheuil (33180)' => '33545', + 'Vervant (16330)' => '16401', + 'Vervant (17400)' => '17467', + 'Veyrac (87520)' => '87202', + 'Veyrières (19200)' => '19283', + 'Veyrignac (24370)' => '24574', + 'Veyrines-de-Domme (24250)' => '24575', + 'Veyrines-de-Vergt (24380)' => '24576', + 'Vézac (24220)' => '24577', + 'Vézières (86120)' => '86287', + 'Vialer (64330)' => '64552', + 'Viam (19170)' => '19284', + 'Vianne (47230)' => '47318', + 'Vibrac (16120)' => '16402', + 'Vibrac (17130)' => '17468', + 'Vicq-d\'Auribat (40380)' => '40324', + 'Vicq-sur-Breuilh (87260)' => '87203', + 'Vicq-sur-Gartempe (86260)' => '86288', + 'Vidaillat (23250)' => '23260', + 'Videix (87600)' => '87204', + 'Vielle-Saint-Girons (40560)' => '40326', + 'Vielle-Soubiran (40240)' => '40327', + 'Vielle-Tursan (40320)' => '40325', + 'Viellenave-d\'Arthez (64170)' => '64554', + 'Viellenave-de-Navarrenx (64190)' => '64555', + 'Vielleségure (64150)' => '64556', + 'Viennay (79200)' => '79347', + 'Viersat (23170)' => '23261', + 'Vieux-Boucau-les-Bains (40480)' => '40328', + 'Vieux-Mareuil (24340)' => '24579', + 'Vieux-Ruffec (16350)' => '16404', + 'Vigeois (19410)' => '19285', + 'Vigeville (23140)' => '23262', + 'Vignes (64410)' => '64557', + 'Vignolles (16300)' => '16405', + 'Vignols (19130)' => '19286', + 'Vignonet (33330)' => '33546', + 'Vilhonneur (16220)' => '16406', + 'Villac (24120)' => '24580', + 'Villamblard (24140)' => '24581', + 'Villandraut (33730)' => '33547', + 'Villard (23800)' => '23263', + 'Villars (24530)' => '24582', + 'Villars-en-Pons (17260)' => '17469', + 'Villars-les-Bois (17770)' => '17470', + 'Villebois-Lavalette (16320)' => '16408', + 'Villebramar (47380)' => '47319', + 'Villedoux (17230)' => '17472', + 'Villefagnan (16240)' => '16409', + 'Villefavard (87190)' => '87206', + 'Villefollet (79170)' => '79348', + 'Villefranche-de-Lonchat (24610)' => '24584', + 'Villefranche-du-Périgord (24550)' => '24585', + 'Villefranche-du-Queyran (47160)' => '47320', + 'Villefranque (64990)' => '64558', + 'Villegats (16700)' => '16410', + 'Villegouge (33141)' => '33548', + 'Villejésus (16140)' => '16411', + 'Villejoubert (16560)' => '16412', + 'Villemain (79110)' => '79349', + 'Villemorin (17470)' => '17473', + 'Villemort (86310)' => '86291', + 'Villenave (40110)' => '40330', + 'Villenave-d\'Ornon (33140)' => '33550', + 'Villenave-de-Rions (33550)' => '33549', + 'Villenave-près-Béarn (65500)' => '65476', + 'Villeneuve (33710)' => '33551', + 'Villeneuve-de-Duras (47120)' => '47321', + 'Villeneuve-de-Marsan (40190)' => '40331', + 'Villeneuve-la-Comtesse (17330)' => '17474', + 'Villeneuve-sur-Lot (47300)' => '47323', + 'Villeréal (47210)' => '47324', + 'Villeton (47400)' => '47325', + 'Villetoureix (24600)' => '24586', + 'Villexavier (17500)' => '17476', + 'Villiers (86190)' => '86292', + 'Villiers-Couture (17510)' => '17477', + 'Villiers-en-Bois (79360)' => '79350', + 'Villiers-en-Plaine (79160)' => '79351', + 'Villiers-le-Roux (16240)' => '16413', + 'Villiers-sur-Chizé (79170)' => '79352', + 'Villognon (16230)' => '16414', + 'Vinax (17510)' => '17478', + 'Vindelle (16430)' => '16415', + 'Viodos-Abense-de-Bas (64130)' => '64559', + 'Virazeil (47200)' => '47326', + 'Virelade (33720)' => '33552', + 'Virollet (17260)' => '17479', + 'Virsac (33240)' => '33553', + 'Virson (17290)' => '17480', + 'Vitrac (24200)' => '24587', + 'Vitrac-Saint-Vincent (16310)' => '16416', + 'Vitrac-sur-Montane (19800)' => '19287', + 'Viven (64450)' => '64560', + 'Viville (16120)' => '16417', + 'Vivonne (86370)' => '86293', + 'Voeuil-et-Giget (16400)' => '16418', + 'Voissay (17400)' => '17481', + 'Vouharte (16330)' => '16419', + 'Vouhé (17700)' => '17482', + 'Vouhé (79310)' => '79354', + 'Vouillé (79230)' => '79355', + 'Vouillé (86190)' => '86294', + 'Voulême (86400)' => '86295', + 'Voulgézac (16250)' => '16420', + 'Voulmentin (79150)' => '79242', + 'Voulon (86700)' => '86296', + 'Vouneuil-sous-Biard (86580)' => '86297', + 'Vouneuil-sur-Vienne (86210)' => '86298', + 'Voutezac (19130)' => '19288', + 'Vouthon (16220)' => '16421', + 'Vouzailles (86170)' => '86299', + 'Vouzan (16410)' => '16422', + 'Xaintrailles (47230)' => '47327', + 'Xaintray (79220)' => '79357', + 'Xambes (16330)' => '16423', + 'Ychoux (40160)' => '40332', + 'Ygos-Saint-Saturnin (40110)' => '40333', + 'Yssandon (19310)' => '19289', + 'Yversay (86170)' => '86300', + 'Yves (17340)' => '17483', + 'Yviers (16210)' => '16424', + 'Yvrac (33370)' => '33554', + 'Yvrac-et-Malleyrand (16110)' => '16425', + 'Yzosse (40180)' => '40334' + ); +} diff --git a/bridges/AutoJMBridge.php b/bridges/AutoJMBridge.php index 598f043..25fb2cb 100644 --- a/bridges/AutoJMBridge.php +++ b/bridges/AutoJMBridge.php @@ -3,63 +3,184 @@ class AutoJMBridge extends BridgeAbstract { const NAME = 'AutoJM'; - const URI = 'http://www.autojm.fr/'; + const URI = 'https://www.autojm.fr/'; const DESCRIPTION = 'Suivre les offres de véhicules proposés par AutoJM en fonction des critères de filtrages'; const MAINTAINER = 'sysadminstory'; const PARAMETERS = array( 'Afficher les offres de véhicules disponible en fonction des critères du site AutoJM' => array( 'url' => array( - 'name' => 'URL de la recherche', + 'name' => 'URL du modèle', 'type' => 'text', 'required' => true, 'title' => 'URL d\'une recherche avec filtre de véhicules sans le http://www.autojm.fr/', - 'exampleValue' => 'gammes/index/398?order_by=finition_asc&energie[]=3&transmission[]=2&dispo=all' + 'exampleValue' => 'achat-voitures-neuves-peugeot-nouvelle-308-5p' + ), + 'energy' => array( + 'name' => 'Carburant', + 'type' => 'list', + 'values' => array( + '-' => '', + 'Diesel' => 1, + 'Essence' => 3, + 'Hybride' => 5 + ), + 'title' => 'Carburant' + ), + 'transmission' => array( + 'name' => 'Transmission', + 'type' => 'list', + 'values' => array( + '-' => '', + 'Automatique' => 1, + 'Manuelle' => 2 + ), + 'title' => 'Transmission' + ), + 'priceMin' => array( + 'name' => 'Prix minimum', + 'type' => 'number', + 'required' => false, + 'title' => 'Prix minimum du véhicule', + 'exampleValue' => '10000', + 'defaultValue' => '0' + ), + 'priceMax' => array( + 'name' => 'Prix maximum', + 'type' => 'number', + 'required' => false, + 'title' => 'Prix maximum du véhicule', + 'exampleValue' => '15000', + 'defaultValue' => '150000' ) ) ); const CACHE_TIMEOUT = 3600; public function getIcon() { - return self::URI . 'assets/images/favicon.ico'; + return self::URI . 'favicon.ico'; + } + + public function getName() { + switch($this->queriedContext) { + case 'Afficher les offres de véhicules disponible en fonction des critères du site AutoJM': + $html = getSimpleHTMLDOMCached(self::URI . $this->getInput('url'), 86400); + $name = html_entity_decode($html->find('title', 0)->plaintext); + return $name; + break; + default: + return parent::getName(); + } + } public function collectData() { - $html = getSimpleHTMLDOM(self::URI . $this->getInput('url')) + + $model_url = self::URI . $this->getInput('url'); + + // Get the session cookies and the form token + $this->getInitialParameters($model_url); + + // Build the form + $post_data = array( + 'form[energy]' => $this->getInput('energy'), + 'form[transmission]' => $this->getInput('transmission'), + 'form[priceMin]' => $this->getInput('priceMin'), + 'form[priceMin]' => $this->getInput('priceMin'), + 'form[_token]' => $this->token + ); + + // Set the Form request content type + $header = array( + 'Content-Type: application/x-www-form-urlencoded; charset=UTF-8', + ); + + // Set the curl options (POST query and content, and session cookies + $curl_opts = array( + CURLOPT_POST => true, + CURLOPT_POSTFIELDS => http_build_query($post_data), + CURLOPT_COOKIE => $this->cookies + ); + + // Get the JSON content of the form + $json = getContents($model_url, $header, $curl_opts) or returnServerError('Could not request AutoJM.'); - $list = $html->find('div[class*=ligne_modele]'); - foreach($list as $element) { - $image = $element->find('img[class=width-100]', 0)->src; - $serie = $element->find('div[class=serie]', 0)->find('span', 0)->plaintext; - $url = $element->find('div[class=serie]', 0)->find('a[class=btn_ligne color-black]', 0)->href; - if($element->find('div[class*=hasStock-info]', 0) != null) { - $dispo = 'Disponible'; - } else { - $dispo = 'Sur commande'; + + // Extract the HTML content from the JSON result + $data = json_decode($json); + $html = str_get_html($data->content); + + // Go through every finisha of the model + $list = $html->find('h3'); + foreach ($list as $finish) { + $finish_name = $finish->plaintext; + $motorizations = $finish->next_sibling()->find('li'); + foreach ($motorizations as $element) { + $image = $element->find('div[class=block-product-image]', 0)->{'data-ga-banner'}; + $serie = $element->find('span[class=model]', 0)->plaintext; + $url = self::URI . substr($element->find('a', 0)->href, 1); + if ($element->find('span[class*=block-product-nbModel]', 0) != null) { + $availability = 'En Stock'; + } else { + $availability = 'Sur commande'; + } + $discount_html = $element->find('span[class*=tag--promo]', 0); + if ($discount_html != null) { + $discount = $discount_html->plaintext; + } else { + $discount = 'inconnue'; + } + $price = $element->find('span[class=price red h1]', 0)->plaintext; + $item = array(); + $item['title'] = $finish_name . ' ' . $serie; + $item['content'] = '

' + . $finish_name . ' ' . $serie . '

'; + $item['content'] .= ''; + + // Add a fictionnal anchor to the RSS element URL, based on the item content ; + // As the URL could be identical even if the price change, some RSS reader will not show those offers as new items + $item['uri'] = $url . '#' . md5($item['content']); + + $this->items[] = $item; + } + } + } + + /** + * Gets the session cookie and the form token + * + * @param string $pageURL The URL from which to get the values + */ + private function getInitialParameters($pageURL) { + $ch = curl_init(); + curl_setopt($ch, CURLOPT_URL, $pageURL); + curl_setopt($ch, CURLOPT_HEADER, true); + curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); + $data = curl_exec($ch); + + // Separate the response header and the content + $headerSize = curl_getinfo($ch, CURLINFO_HEADER_SIZE); + $header = substr($data, 0, $headerSize); + $content = substr($data, $headerSize); + curl_close($ch); + + // Extract the cookies from the headers + $cookies = ''; + $http_response_header = explode("\r\n", $header); + foreach ($http_response_header as $hdr) { + if (strpos($hdr, 'Set-Cookie') !== false) { + $cLine = explode(':', $hdr)[1]; + $cLine = explode(';', $cLine)[0]; + $cookies .= ';' . $cLine; } - $carburant = str_replace('dispo |', '', $element->find('div[class=carburant]', 0)->plaintext); - $transmission = $element->find('div[class*=bv]', 0)->plaintext; - $places = $element->find('div[class*=places]', 0)->plaintext; - $portes = $element->find('div[class*=nb_portes]', 0)->plaintext; - $carosserie = $element->find('div[class*=coloris]', 0)->plaintext; - $remise = $element->find('div[class*=remise]', 0)->plaintext; - $prix = $element->find('div[class*=prixjm]', 0)->plaintext; - - $item = array(); - $item['uri'] = $url; - $item['title'] = $serie; - $item['content'] = '

' . $serie . '

'; - $item['content'] .= ''; - - $this->items[] = $item; } + $this->cookies = trim(substr($cookies, 1)); + // Get the token from the content + $html = str_get_html($content); + $token = $html->find('input[type=hidden][id=form__token]', 0); + $this->token = $token->value; } } diff --git a/bridges/BAEBridge.php b/bridges/BAEBridge.php index caa2cf7..6c5d8ba 100644 --- a/bridges/BAEBridge.php +++ b/bridges/BAEBridge.php @@ -55,9 +55,7 @@ class BAEBridge extends BridgeAbstract { $content .= '
'; $content .= $htmlDetail->find('section', 0)->innertext; - $content = str_replace('src="/', 'src="' . parent::getURI() . '/', $content); - $content = str_replace('href="/', 'href="' . parent::getURI() . '/', $content); - $item['content'] = $content; + $item['content'] = defaultLinkTo($content, parent::getURI()); $image = $htmlDetail->find('#zoom', 0); if ($image) { $item['enclosures'] = array(parent::getURI() . $image->getAttribute('src')); diff --git a/bridges/BadDragonBridge.php b/bridges/BadDragonBridge.php new file mode 100644 index 0000000..d606c4e --- /dev/null +++ b/bridges/BadDragonBridge.php @@ -0,0 +1,435 @@ + array( + ), + 'Clearance' => array( + 'ready_made' => array( + 'name' => 'Ready Made', + 'type' => 'checkbox' + ), + 'flop' => array( + 'name' => 'Flops', + 'type' => 'checkbox' + ), + 'skus' => array( + 'name' => 'Products', + 'exampleValue' => 'chanceflared, crackers', + 'title' => 'Comma separated list of product SKUs' + ), + 'onesize' => array( + 'name' => 'One-Size', + 'type' => 'checkbox' + ), + 'mini' => array( + 'name' => 'Mini', + 'type' => 'checkbox' + ), + 'small' => array( + 'name' => 'Small', + 'type' => 'checkbox' + ), + 'medium' => array( + 'name' => 'Medium', + 'type' => 'checkbox' + ), + 'large' => array( + 'name' => 'Large', + 'type' => 'checkbox' + ), + 'extralarge' => array( + 'name' => 'Extra Large', + 'type' => 'checkbox' + ), + 'category' => array( + 'name' => 'Category', + 'type' => 'list', + 'values' => array( + 'All' => 'all', + 'Accessories' => 'accessories', + 'Merchandise' => 'merchandise', + 'Dildos' => 'insertable', + 'Masturbators' => 'penetrable', + 'Packers' => 'packer', + 'Lil\' Squirts' => 'shooter', + 'Lil\' Vibes' => 'vibrator', + 'Wearables' => 'wearable' + ), + 'defaultValue' => 'all', + ), + 'soft' => array( + 'name' => 'Soft Firmness', + 'type' => 'checkbox' + ), + 'med_firm' => array( + 'name' => 'Medium Firmness', + 'type' => 'checkbox' + ), + 'firm' => array( + 'name' => 'Firm', + 'type' => 'checkbox' + ), + 'split' => array( + 'name' => 'Split Firmness', + 'type' => 'checkbox' + ), + 'maxprice' => array( + 'name' => 'Max Price', + 'type' => 'number', + 'required' => true, + 'defaultValue' => 300 + ), + 'minprice' => array( + 'name' => 'Min Price', + 'type' => 'number', + 'defaultValue' => 0 + ), + 'cumtube' => array( + 'name' => 'Cumtube', + 'type' => 'checkbox' + ), + 'suctionCup' => array( + 'name' => 'Suction Cup', + 'type' => 'checkbox' + ), + 'noAccessories' => array( + 'name' => 'No Accessories', + 'type' => 'checkbox' + ) + ) + ); + + /* + * This sets index $strFrom (or $strTo if set) in $outArr to 'on' if + * $inArr[$param] contains $strFrom. + * It is used for translating BD's shop filter URLs into something we can use. + * + * For the query '?type[]=ready_made&type[]=flop' we would have an array like: + * Array ( + * [type] => Array ( + * [0] => ready_made + * [1] => flop + * ) + * ) + * which could be translated into: + * Array ( + * [ready_made] => on + * [flop] => on + * ) + * */ + private function setParam($inArr, &$outArr, $param, $strFrom, $strTo = null) { + if(isset($inArr[$param]) && in_array($strFrom, $inArr[$param])) { + $outArr[($strTo ?: $strFrom)] = 'on'; + } + } + + public function detectParameters($url) { + $params = array(); + + // Sale + $regex = '/^(https?:\/\/)?bad-dragon\.com\/sales/'; + if(preg_match($regex, $url, $matches) > 0) { + return $params; + } + + // Clearance + $regex = '/^(https?:\/\/)?bad-dragon\.com\/shop\/clearance/'; + if(preg_match($regex, $url, $matches) > 0) { + parse_str(parse_url($url, PHP_URL_QUERY), $urlParams); + + $this->setParam($urlParams, $params, 'type', 'ready_made'); + $this->setParam($urlParams, $params, 'type', 'flop'); + + if(isset($urlParams['skus'])) { + $skus = array(); + foreach($urlParams['skus'] as $sku) { + is_string($sku) && $skus[] = $sku; + is_array($sku) && $skus[] = $sku[0]; + } + $params['skus'] = implode(',', $skus); + } + + $this->setParam($urlParams, $params, 'sizes', 'onesize'); + $this->setParam($urlParams, $params, 'sizes', 'mini'); + $this->setParam($urlParams, $params, 'sizes', 'small'); + $this->setParam($urlParams, $params, 'sizes', 'medium'); + $this->setParam($urlParams, $params, 'sizes', 'large'); + $this->setParam($urlParams, $params, 'sizes', 'extralarge'); + + if(isset($urlParams['category'])) { + $params['category'] = strtolower($urlParams['category']); + } else{ + $params['category'] = 'all'; + } + + $this->setParam($urlParams, $params, 'firmnessValues', 'soft'); + $this->setParam($urlParams, $params, 'firmnessValues', 'medium', 'med_firm'); + $this->setParam($urlParams, $params, 'firmnessValues', 'firm'); + $this->setParam($urlParams, $params, 'firmnessValues', 'split'); + + if(isset($urlParams['price'])) { + isset($urlParams['price']['max']) + && $params['maxprice'] = $urlParams['price']['max']; + isset($urlParams['price']['min']) + && $params['minprice'] = $urlParams['price']['min']; + } + + isset($urlParams['cumtube']) + && $urlParams['cumtube'] === '1' + && $params['cumtube'] = 'on'; + isset($urlParams['suctionCup']) + && $urlParams['suctionCup'] === '1' + && $params['suctionCup'] = 'on'; + isset($urlParams['noAccessories']) + && $urlParams['noAccessories'] === '1' + && $params['noAccessories'] = 'on'; + + return $params; + } + + return null; + } + + public function getName() { + switch($this->queriedContext) { + case 'Sales': + return 'Bad Dragon Sales'; + case 'Clearance': + return 'Bad Dragon Clearance Search'; + default: + return parent::getName(); + } + } + + public function getURI() { + switch($this->queriedContext) { + case 'Sales': + return self::URI . 'sales'; + case 'Clearance': + return $this->inputToURL(); + default: + return parent::getURI(); + } + } + + public function collectData() { + switch($this->queriedContext) { + case 'Sales': + $sales = json_decode(getContents(self::URI . 'api/sales')) + or returnServerError('Failed to query BD API'); + + foreach($sales as $sale) { + $item = array(); + + $item['title'] = $sale->title; + $item['timestamp'] = strtotime($sale->startDate); + + $item['uri'] = $this->getURI() . '/' . $sale->slug; + + $contentHTML = '

'; + if(isset($sale->endDate)) { + $contentHTML .= '

This promotion ends on ' + . gmdate('M j, Y \a\t g:i A T', strtotime($sale->endDate)) + . '

'; + } else { + $contentHTML .= '

This promotion never ends

'; + } + $ul = false; + $content = json_decode($sale->content); + foreach($content->blocks as $block) { + switch($block->type) { + case 'header-one': + $contentHTML .= '

' . $block->text . '

'; + break; + case 'header-two': + $contentHTML .= '

' . $block->text . '

'; + break; + case 'header-three': + $contentHTML .= '

' . $block->text . '

'; + break; + case 'unordered-list-item': + if(!$ul) { + $contentHTML .= ''; + $ul = false; + } + $contentHTML .= '

' . $block->text . '

'; + break; + } + } + $item['content'] = $contentHTML; + + $this->items[] = $item; + } + break; + case 'Clearance': + $toyData = json_decode(getContents($this->inputToURL(true))) + or returnServerError('Failed to query BD API'); + + $productList = json_decode(getContents(self::URI + . 'api/inventory-toy/product-list')) + or returnServerError('Failed to query BD API'); + + foreach($toyData->toys as $toy) { + $item = array(); + + $item['uri'] = $this->getURI() + . '#' + . $toy->id; + $item['timestamp'] = strtotime($toy->created); + + foreach($productList as $product) { + if($product->sku == $toy->sku) { + $item['title'] = $product->name; + break; + } + } + + // images + $content = '

'; + foreach($toy->images as $image) { + $content .= ''; + } + // price + $content .= '

Price: $' + . $toy->price + // size + . '
Size: ' + . $toy->size + // color + . '
Color: ' + . $toy->color + // features + . '
Features: ' + . ($toy->suction_cup ? 'Suction cup' : '') + . ($toy->suction_cup && $toy->cumtube ? ', ' : '') + . ($toy->cumtube ? 'Cumtube' : '') + . ($toy->suction_cup || $toy->cumtube ? '' : 'None'); + // firmness + $firmnessTexts = array( + '2' => 'Extra soft', + '3' => 'Soft', + '5' => 'Medium', + '8' => 'Firm' + ); + $firmnesses = explode('/', $toy->firmness); + if(count($firmnesses) === 2) { + $content .= '
Firmness: ' + . $firmnessTexts[$firmnesses[0]] + . ', ' + . $firmnessTexts[$firmnesses[1]]; + } else{ + $content .= '
Firmness: ' + . $firmnessTexts[$firmnesses[0]]; + } + // flop + if($toy->type === 'flop') { + $content .= '
Flop reason: ' + . $toy->flop_reason; + } + $content .= '

'; + $item['content'] = $content; + + $enclosures = array(); + foreach($toy->images as $image) { + $enclosures[] = $image->fullFilename; + } + $item['enclosures'] = $enclosures; + + $categories = array(); + $categories[] = $toy->sku; + $categories[] = $toy->type; + $categories[] = $toy->size; + if($toy->cumtube) { + $categories[] = 'cumtube'; + } + if($toy->suction_cup) { + $categories[] = 'suction_cup'; + } + $item['categories'] = $categories; + + $this->items[] = $item; + } + break; + } + } + + private function inputToURL($api = false) { + $url = self::URI; + $url .= ($api ? 'api/inventory-toys?' : 'shop/clearance?'); + + // Default parameters + $url .= 'limit=60'; + $url .= '&page=1'; + $url .= '&sort[field]=created'; + $url .= '&sort[direction]=desc'; + + // Product types + $url .= ($this->getInput('ready_made') ? '&type[]=ready_made' : ''); + $url .= ($this->getInput('flop') ? '&type[]=flop' : ''); + + // Product names + foreach(array_filter(explode(',', $this->getInput('skus'))) as $sku) { + $url .= '&skus[]=' . urlencode(trim($sku)); + } + + // Size + $url .= ($this->getInput('onesize') ? '&sizes[]=onesize' : ''); + $url .= ($this->getInput('mini') ? '&sizes[]=mini' : ''); + $url .= ($this->getInput('small') ? '&sizes[]=small' : ''); + $url .= ($this->getInput('medium') ? '&sizes[]=medium' : ''); + $url .= ($this->getInput('large') ? '&sizes[]=large' : ''); + $url .= ($this->getInput('extralarge') ? '&sizes[]=extralarge' : ''); + + // Category + $url .= ($this->getInput('category') ? '&category=' + . urlencode($this->getInput('category')) : ''); + + // Firmness + if($api) { + $url .= ($this->getInput('soft') ? '&firmnessValues[]=3' : ''); + $url .= ($this->getInput('med_firm') ? '&firmnessValues[]=5' : ''); + $url .= ($this->getInput('firm') ? '&firmnessValues[]=8' : ''); + if($this->getInput('split')) { + $url .= '&firmnessValues[]=3/5'; + $url .= '&firmnessValues[]=3/8'; + $url .= '&firmnessValues[]=8/3'; + $url .= '&firmnessValues[]=5/8'; + $url .= '&firmnessValues[]=8/5'; + } + } else{ + $url .= ($this->getInput('soft') ? '&firmnessValues[]=soft' : ''); + $url .= ($this->getInput('med_firm') ? '&firmnessValues[]=medium' : ''); + $url .= ($this->getInput('firm') ? '&firmnessValues[]=firm' : ''); + $url .= ($this->getInput('split') ? '&firmnessValues[]=split' : ''); + } + + // Price + $url .= ($this->getInput('maxprice') ? '&price[max]=' + . $this->getInput('maxprice') : '&price[max]=300'); + $url .= ($this->getInput('minprice') ? '&price[min]=' + . $this->getInput('minprice') : '&price[min]=0'); + + // Features + $url .= ($this->getInput('cumtube') ? '&cumtube=1' : ''); + $url .= ($this->getInput('suctionCup') ? '&suctionCup=1' : ''); + $url .= ($this->getInput('noAccessories') ? '&noAccessories=1' : ''); + + return $url; + } +} diff --git a/bridges/BakaUpdatesMangaReleasesBridge.php b/bridges/BakaUpdatesMangaReleasesBridge.php new file mode 100644 index 0000000..27eca28 --- /dev/null +++ b/bridges/BakaUpdatesMangaReleasesBridge.php @@ -0,0 +1,103 @@ + array( + 'name' => 'Series ID', + 'type' => 'number', + 'required' => true, + 'exampleValue' => '12345' + ) + )); + const LIMIT_COLS = 5; + const LIMIT_ITEMS = 10; + + private $feedName = ''; + + public function collectData() { + $html = getSimpleHTMLDOM($this->getURI()) + or returnServerError('Series not found'); + + // content is an unstructured pile of divs, ugly to parse + $cols = $html->find('div#main_content div.row > div.text'); + if (!$cols) + returnServerError('No releases'); + + $rows = array_slice( + array_chunk($cols, self::LIMIT_COLS), 0, self::LIMIT_ITEMS + ); + + if (isset($rows[0][1])) { + $this->feedName = $this->filterHTML($rows[0][1]->plaintext); + } + + foreach($rows as $cols) { + if (count($cols) < self::LIMIT_COLS) continue; + + $item = array(); + $title = array(); + + $item['content'] = ''; + + $objDate = $cols[0]; + if ($objDate) + $item['timestamp'] = strtotime($objDate->plaintext); + + $objTitle = $cols[1]; + if ($objTitle) { + $title[] = $this->filterHTML($objTitle->plaintext); + $item['content'] .= '

Series: ' . $this->filterText($objTitle->innertext) . '

'; + } + + $objVolume = $cols[2]; + if ($objVolume && !empty($objVolume->plaintext)) + $title[] = 'Vol.' . $objVolume->plaintext; + + $objChapter = $cols[3]; + if ($objChapter && !empty($objChapter->plaintext)) + $title[] = 'Chp.' . $objChapter->plaintext; + + $objAuthor = $cols[4]; + if ($objAuthor && !empty($objAuthor->plaintext)) { + $item['author'] = $this->filterHTML($objAuthor->plaintext); + $item['content'] .= '

Groups: ' . $this->filterText($objAuthor->innertext) . '

'; + } + + $item['title'] = implode(' ', $title); + $item['uri'] = $this->getURI(); + $item['uid'] = $this->getSanitizedHash($item['title']); + + $this->items[] = $item; + } + } + + public function getURI(){ + $series_id = $this->getInput('series_id'); + if (!empty($series_id)) { + return self::URI . 'releases.html?search=' . $series_id . '&stype=series'; + } + return self::URI; + } + + public function getName(){ + if(!empty($this->feedName)) { + return $this->feedName . ' - ' . self::NAME; + } + return parent::getName(); + } + + private function getSanitizedHash($string) { + return hash('sha1', preg_replace('/[^a-zA-Z0-9\-\.]/', '', ucwords(strtolower($string)))); + } + + private function filterText($text) { + return rtrim($text, '* '); + } + + private function filterHTML($text) { + return $this->filterText(html_entity_decode($text)); + } +} diff --git a/bridges/BandcampBridge.php b/bridges/BandcampBridge.php index 9c8d436..6c75ed5 100644 --- a/bridges/BandcampBridge.php +++ b/bridges/BandcampBridge.php @@ -13,48 +13,72 @@ class BandcampBridge extends BridgeAbstract { 'required' => true ) )); + const IMGURI = 'https://f4.bcbits.com/'; + const IMGSIZE_300PX = 23; + const IMGSIZE_700PX = 16; public function getIcon() { return 'https://s4.bcbits.com/img/bc_favicon.ico'; } public function collectData(){ - $html = getSimpleHTMLDOM($this->getURI()) - or returnServerError('No results for this query.'); + $url = self::URI . 'api/hub/1/dig_deeper'; + $data = $this->buildRequestJson(); + $header = array( + 'Content-Type: application/json', + 'Content-Length: ' . strlen($data) + ); + $opts = array( + CURLOPT_CUSTOMREQUEST => 'POST', + CURLOPT_POSTFIELDS => $data + ); + $content = getContents($url, $header, $opts) + or returnServerError('Could not complete request to: ' . $url); - foreach($html->find('li.item') as $release) { - $script = $release->find('div.art', 0)->getAttribute('onclick'); - $uri = ltrim($script, "return 'url("); - $uri = rtrim($uri, "')"); + $json = json_decode($content); - $item = array(); - $item['author'] = $release->find('div.itemsubtext', 0)->plaintext - . ' - ' - . $release->find('div.itemtext', 0)->plaintext; + if ($json->ok !== true) { + returnServerError('Invalid response'); + } - $item['title'] = $release->find('div.itemsubtext', 0)->plaintext - . ' - ' - . $release->find('div.itemtext', 0)->plaintext; + foreach ($json->items as $entry) { + $url = $entry->tralbum_url; + $artist = $entry->artist; + $title = $entry->title; + // e.g. record label is the releaser, but not the artist + $releaser = $entry->band_name !== $entry->artist ? $entry->band_name : null; - $item['content'] = '
' - . $release->find('div.itemsubtext', 0)->plaintext - . ' - ' - . $release->find('div.itemtext', 0)->plaintext; + $full_title = $artist . ' - ' . $title; + $full_artist = $artist; + if (isset($releaser)) { + $full_title .= ' (' . $releaser . ')'; + $full_artist .= ' (' . $releaser . ')'; + } + $small_img = $this->getImageUrl($entry->art_id, self::IMGSIZE_300PX); + $img = $this->getImageUrl($entry->art_id, self::IMGSIZE_700PX); - $item['id'] = $release->find('a', 0)->getAttribute('href'); - $item['uri'] = $release->find('a', 0)->getAttribute('href'); + $item = array( + 'uri' => $url, + 'author' => $full_artist, + 'title' => $full_title + ); + $item['content'] = "
$full_title"; + $item['enclosures'] = array($img); $this->items[] = $item; } } - public function getURI(){ - if(!is_null($this->getInput('tag'))) { - return self::URI . 'tag/' . urlencode($this->getInput('tag')) . '?sort_field=date'; - } + private function buildRequestJson(){ + $requestJson = array( + 'tag' => $this->getInput('tag'), + 'page' => 1, + 'sort' => 'date' + ); + return json_encode($requestJson); + } - return parent::getURI(); + private function getImageUrl($id, $size){ + return self::IMGURI . 'img/a' . $id . '_' . $size . '.jpg'; } public function getName(){ diff --git a/bridges/BinanceBridge.php b/bridges/BinanceBridge.php new file mode 100644 index 0000000..9653ab7 --- /dev/null +++ b/bridges/BinanceBridge.php @@ -0,0 +1,103 @@ + array( + 'name' => 'category', + 'type' => 'list', + 'exampleValue' => 'Blog', + 'title' => 'Select a category', + 'values' => array( + 'Blog' => 'Blog', + 'Announcements' => 'Announcements' + ) + ) + )); + + public function getIcon() { + return 'https://bin.bnbstatic.com/static/images/common/favicon.ico'; + } + + public function getName() { + return self::NAME . ' ' . $this->getInput('category'); + } + + public function getURI() { + if ($this->getInput('category') == 'Blog') + return self::URI . '/en/blog'; + else + return 'https://binance.zendesk.com/hc/en-us/categories/115000056351-Announcements'; + } + + protected function collectBlogData() { + $html = getSimpleHTMLDOM($this->getURI()) + or returnServerError('Could not fetch Binance blog data.'); + + foreach($html->find('div[direction="row"]') as $element) { + + $date = $element->find('div[direction="column"]', 0); + $day = $date->find('div', 0)->innertext; + $month = $date->find('div', 1)->innertext; + $extractedDate = $day . ' ' . $month; + + $abstract = $element->find('div[direction="column"]', 1); + $a = $abstract->find('a', 0); + $uri = self::URI . $a->href; + $title = $a->innertext; + + $full = getSimpleHTMLDOMCached($uri); + $content = $full->find('div.desc', 1); + + $item = array(); + $item['title'] = $title; + $item['uri'] = $uri; + $item['timestamp'] = strtotime($extractedDate); + $item['author'] = 'Binance'; + $item['content'] = $content; + + $this->items[] = $item; + + if (count($this->items) >= 10) + break; + } + } + + protected function collectAnnouncementData() { + $html = getSimpleHTMLDOM($this->getURI()) + or returnServerError('Could not fetch Zendesk announcement data.'); + + foreach($html->find('a.article-list-link') as $a) { + $title = $a->innertext; + $uri = 'https://binance.zendesk.com' . $a->href; + + $full = getSimpleHTMLDOMCached($uri); + $content = $full->find('div.article-body', 0); + $date = $full->find('time', 0)->getAttribute('datetime'); + + $item = array(); + + $item['title'] = $title; + $item['uri'] = $uri; + $item['timestamp'] = strtotime($date); + $item['author'] = 'Binance'; + $item['content'] = $content; + + $this->items[] = $item; + + if (count($this->items) >= 10) + break; + } + } + + public function collectData() { + if ($this->getInput('category') == 'Blog') + $this->collectBlogData(); + else + $this->collectAnnouncementData(); + } +} diff --git a/bridges/BingSearchBridge.php b/bridges/BingSearchBridge.php new file mode 100644 index 0000000..eb8a5fc --- /dev/null +++ b/bridges/BingSearchBridge.php @@ -0,0 +1,119 @@ + array( + 'category' => array( + 'name' => 'Categories', + 'type' => 'list', + 'values' => self::IMAGE_DISCOVER_CATEGORIES + ), + 'image_size' => array( + 'name' => 'Image size', + 'type' => 'list', + 'values' => array( + 'Small' => 'turl', + 'Full size' => 'imgurl' + ) + ) + ) + ); + + const IMAGE_DISCOVER_CATEGORIES = array( + 'Abstract' => 'abstract', + 'Animals' => 'animals', + 'Anime' => 'anime', + 'Architecture' => 'architecture', + 'Arts and Crafts' => 'arts-and-crafts', + 'Beauty' => 'beauty', + 'Cars and Motorcycles' => 'cars-and-motorcycles', + 'Cats' => 'cats', + 'Celebrities' => 'celebrities', + 'Comics' => 'comics', + 'DIY' => 'diy', + 'Dogs' => 'dogs', + 'Fitness' => 'fitness', + 'Food and Drink' => 'food-and-drink', + 'Funny' => 'funny', + 'Gadgets' => 'gadgets', + 'Gardening' => 'gardening', + 'Geeky' => 'geeky', + 'Hairstyles' => 'hairstyles', + 'Home Decor' => 'home-decor', + 'Marine Life' => 'marine-life', + 'Men\'s Fashion' => 'men%27s-fashion', + 'Nature' => 'nature', + 'Outdoors' => 'outdoors', + 'Parenting' => 'parenting', + 'Phone Wallpapers' => 'phone-wallpapers', + 'Photography' => 'photography', + 'Quotes' => 'quotes', + 'Recipes' => 'recipes', + 'Snow' => 'snow', + 'Tattoos' => 'tattoos', + 'Travel' => 'travel', + 'Video Games' => 'video-games', + 'Weddings' => 'weddings', + 'Women\'s Fashion' => 'women%27s-fashion', + ); + + public function getIcon() + { + return 'https://www.bing.com/sa/simg/bing_p_rr_teal_min.ico'; + } + + public function collectData() + { + $this->items = $this->imageDiscover($this->getInput('category')); + } + + public function getName() + { + if ($this->getInput('category')) { + if (self::IMAGE_DISCOVER_CATEGORIES[$this->getInput('categories')] !== null) { + $category = self::IMAGE_DISCOVER_CATEGORIES[$this->getInput('categories')]; + } else { + $category = 'Unknown'; + } + + return 'Best ' . $category . ' - Bing Image Discover'; + } + return parent::getName(); + } + + private function imageDiscover($category) + { + $html = getSimpleHTMLDOM(self::URI . '/discover/' . $category) + or returnServerError('Could not request ' . self::NAME); + $sizeKey = $this->getInput('image_size'); + + $items = []; + foreach ($html->find('a.iusc') as $element) { + $data = json_decode(htmlspecialchars_decode($element->getAttribute('m')), true); + + $item = array(); + $item['title'] = basename(rtrim($data['imgurl'], '/')); + $item['uri'] = $data['imgurl']; + $item['content'] = ' + ' . $item['title'] . ' +

Source:

'; + $item['enclosures'] = $data['imgurl']; + + $items[] = $item; + } + return $items; + } + + private function curUrl($url) + { + if (strlen($url) <= 80) { + return $url; + } + return substr($url, 0, 80) . '...'; + } +} diff --git a/bridges/BrutBridge.php b/bridges/BrutBridge.php new file mode 100644 index 0000000..32265b6 --- /dev/null +++ b/bridges/BrutBridge.php @@ -0,0 +1,157 @@ + array( + 'name' => 'Category', + 'type' => 'list', + 'values' => array( + 'News' => 'news', + 'International' => 'international', + 'Economy' => 'economy', + 'Science and Technology' => 'science-and-technology', + 'Entertainment' => 'entertainment', + 'Sports' => 'sport', + 'Nature' => 'nature', + ), + 'defaultValue' => 'news', + ), + 'edition' => array( + 'name' => ' Edition', + 'type' => 'list', + 'values' => array( + 'United States' => 'us', + 'United Kingdom' => 'uk', + 'France' => 'fr', + 'India' => 'in', + 'Mexico' => 'mx', + ), + 'defaultValue' => 'us', + ) + ) + ); + + const CACHE_TIMEOUT = 1800; // 30 mins + + private $videoId = ''; + private $videoType = ''; + private $videoImage = ''; + + public function collectData() { + + $html = getSimpleHTMLDOM($this->getURI()) + or returnServerError('Could not request: ' . $this->getURI()); + + $results = $html->find('div.results', 0); + + foreach($results->find('li.col-6.col-sm-4.col-md-3.col-lg-2.px-2.pb-4') as $index => $li) { + $item = array(); + + $videoPath = self::URI . $li->children(0)->href; + + $videoPageHtml = getSimpleHTMLDOMCached($videoPath, 3600) + or returnServerError('Could not request: ' . $videoPath); + + $this->videoImage = $videoPageHtml->find('meta[name="twitter:image"]', 0)->content; + + $this->processTwitterImage(); + + $description = $videoPageHtml->find('div.description', 0); + + $item['uri'] = $videoPath; + $item['title'] = $description->find('h1', 0)->plaintext; + + if ($description->find('div.date', 0)->children(0)) { + $description->find('div.date', 0)->children(0)->outertext = ''; + } + + $item['content'] = $this->processContent( + $description + ); + + $item['timestamp'] = $this->processDate($description); + $item['enclosures'][] = $this->videoImage; + + $this->items[] = $item; + + if (count($this->items) >= 5) { + break; + } + } + } + + public function getURI() { + + if (!is_null($this->getInput('edition')) && !is_null($this->getInput('category'))) { + return self::URI . '/' . $this->getInput('edition') . '/' . $this->getInput('category'); + } + + return parent::getURI(); + } + + public function getName() { + + if (!is_null($this->getInput('edition')) && !is_null($this->getInput('category'))) { + $parameters = $this->getParameters(); + + $editionValues = array_flip($parameters[0]['edition']['values']); + $categoryValues = array_flip($parameters[0]['category']['values']); + + return $categoryValues[$this->getInput('category')] . ' - ' . + $editionValues[$this->getInput('edition')] . ' - Brut.'; + } + + return parent::getName(); + } + + private function processDate($description) { + + if ($this->getInput('edition') === 'uk') { + $date = DateTime::createFromFormat('d/m/Y H:i', $description->find('div.date', 0)->innertext); + return strtotime($date->format('Y-m-d H:i:s')); + } + + return strtotime($description->find('div.date', 0)->innertext); + } + + private function processContent($description) { + + $content = ''; + $content .= '

' . $description->find('h2.mb-1', 0)->innertext . '

'; + + if ($description->find('div.text.pb-3', 0)->children(1)->class != 'date') { + $content .= '

' . $description->find('div.text.pb-3', 0)->children(1)->innertext . '

'; + } + + return $content; + } + + private function processTwitterImage() { + /** + * Extract video ID + type from twitter image + * + * Example (wrapped): + * https://img.brut.media/thumbnail/ + * the-life-of-rita-moreno-2cce75b5-d448-44d2-a97c-ca50d6470dd4-square.jpg + * ?ts=1559337892 + */ + $fpath = parse_url($this->videoImage, PHP_URL_PATH); + $fname = basename($fpath); + $fname = substr($fname, 0, strrpos($fname, '.')); + $parts = explode('-', $fname); + + if (end($parts) === 'auto') { + $key = array_search('auto', $parts); + unset($parts[$key]); + } + + $this->videoId = implode('-', array_splice($parts, -6, 5)); + $this->videoType = end($parts); + } +} diff --git a/bridges/BundesbankBridge.php b/bridges/BundesbankBridge.php index d21f22b..b64a642 100644 --- a/bridges/BundesbankBridge.php +++ b/bridges/BundesbankBridge.php @@ -17,7 +17,6 @@ class BundesbankBridge extends BridgeAbstract { self::PARAM_LANG => array( 'name' => 'Language', 'type' => 'list', - 'required' => true, 'defaultValue' => self::LANG_DE, 'values' => array( 'English' => self::LANG_EN, diff --git a/bridges/CNETFranceBridge.php b/bridges/CNETFranceBridge.php new file mode 100644 index 0000000..222c8b9 --- /dev/null +++ b/bridges/CNETFranceBridge.php @@ -0,0 +1,63 @@ + array( + 'title' => array( + 'name' => 'Exclude by title', + 'required' => false, + 'title' => 'Title term, separated by semicolon (;)', + 'defaultValue' => 'bon plan;bons plans;au meilleur prix;des meilleures offres;Amazon Prime Day;RED by SFR ou B&You' + ), + 'url' => array( + 'name' => 'Exclude by url', + 'required' => false, + 'title' => 'URL term, separated by semicolon (;)', + 'defaultValue' => 'bon-plan;bons-plans' + ) + ) + ); + + private $bannedTitle = []; + private $bannedURL = []; + + public function collectData() + { + $title = $this->getInput('title'); + $url = $this->getInput('url'); + + if ($title !== null) { + $this->bannedTitle = explode(';', $title); + } + + if ($url !== null) { + $this->bannedURL = explode(';', $url); + } + + $this->collectExpandableDatas('https://www.cnetfrance.fr/feeds/rss/news/'); + } + + protected function parseItem($feedItem) + { + $item = parent::parseItem($feedItem); + + foreach ($this->bannedTitle as $term) { + if (preg_match('/' . $term . '/mi', $item['title']) === 1) { + return null; + } + } + + foreach ($this->bannedURL as $term) { + if (preg_match('/' . $term . '/mi', $item['uri']) === 1) { + return null; + } + } + + return $item; + } +} diff --git a/bridges/CachetBridge.php b/bridges/CachetBridge.php new file mode 100644 index 0000000..a60b8f7 --- /dev/null +++ b/bridges/CachetBridge.php @@ -0,0 +1,134 @@ + array( + 'name' => 'Cachet installation', + 'type' => 'text', + 'required' => true, + 'title' => 'The URL of the Cachet installation', + 'exampleValue' => 'https://demo.cachethq.io/', + ), 'additional_info' => array( + 'name' => 'Additional Timestamps', + 'type' => 'checkbox', + 'title' => 'Whether to include the given timestamps' + ) + ) + ); + const CACHE_TIMEOUT = 300; + + private $componentCache = []; + + public function getURI() { + return $this->getInput('host') === null ? 'https://cachethq.io/' : $this->getInput('host'); + } + + /** + * Validates the ping request to the cache API + * + * @param string $ping + * @return boolean + */ + private function validatePing($ping) { + $ping = json_decode($ping); + if ($ping === null) { + return false; + } + return $ping->data === 'Pong!'; + } + + /** + * Returns the component name of a cachat component + * + * @param integer $id + * @return string + */ + private function getComponentName($id) { + if ($id === 0) { + return ''; + } + if (array_key_exists($id, $this->componentCache)) { + return $this->componentCache[$id]; + } + + $component = getContents($this->getURI() . '/api/v1/components/' . $id); + $component = json_decode($component); + if ($component === null) { + return ''; + } + return $component->data->name; + } + + public function collectData() { + $ping = getContents(urljoin($this->getURI(), '/api/v1/ping')); + if (!$this->validatePing($ping)) { + returnClientError('Provided URI is invalid!'); + } + + $url = urljoin($this->getURI(), '/api/v1/incidents?sort=id&order=desc'); + $incidents = getContents($url); + $incidents = json_decode($incidents); + if ($incidents === null) { + returnClientError('/api/v1/incidents returned no valid json'); + } + + usort($incidents->data, function ($a, $b) { + $timeA = strtotime($a->updated_at); + $timeB = strtotime($b->updated_at); + return $timeA > $timeB ? -1 : 1; + }); + + foreach ($incidents->data as $incident) { + + if (isset($incident->permalink)) { + $permalink = $incident->permalink; + } else { + $permalink = urljoin($this->getURI(), '/incident/' . $incident->id); + } + + $title = $incident->human_status . ': ' . $incident->name; + $message = ''; + if ($this->getInput('additional_info')) { + if (isset($incident->occurred_at)) { + $message .= 'Occurred at: ' . $incident->occurred_at . "\r\n"; + } + if (isset($incident->scheduled_at)) { + $message .= 'Scheduled at: ' . $incident->scheduled_at . "\r\n"; + } + if (isset($incident->created_at)) { + $message .= 'Created at: ' . $incident->created_at . "\r\n"; + } + if (isset($incident->updated_at)) { + $message .= 'Updated at: ' . $incident->updated_at . "\r\n\r\n"; + } + } + + $message .= $incident->message; + $content = nl2br($message); + $componentName = $this->getComponentName($incident->component_id); + $uidOrig = $permalink . $incident->created_at; + $uid = hash('sha512', $uidOrig); + $timestamp = strtotime($incident->created_at); + $categories = []; + $categories[] = $incident->human_status; + if ($componentName !== '') { + $categories[] = $componentName; + } + + $item = []; + $item['uri'] = $permalink; + $item['title'] = $title; + $item['timestamp'] = $timestamp; + $item['content'] = $content; + $item['uid'] = $uid; + $item['categories'] = $categories; + + $this->items[] = $item; + } + } +} diff --git a/bridges/CastorusBridge.php b/bridges/CastorusBridge.php index 3ed1331..c394283 100644 --- a/bridges/CastorusBridge.php +++ b/bridges/CastorusBridge.php @@ -83,7 +83,7 @@ class CastorusBridge extends BridgeAbstract { if(!$html) returnServerError('Could not load data from ' . self::URI . '!'); - $activities = $html->find('div#activite/li'); + $activities = $html->find('div#activite > li'); if(!$activities) returnServerError('Failed to find activities!'); diff --git a/bridges/ComboiosDePortugalBridge.php b/bridges/ComboiosDePortugalBridge.php new file mode 100644 index 0000000..610e23b --- /dev/null +++ b/bridges/ComboiosDePortugalBridge.php @@ -0,0 +1,22 @@ +getURI() . '/consultar-horarios/avisos') + or returnServerError('Could not load content'); + + foreach($html->find('.warnings-table a') as $element) { + $item = array(); + + $item['title'] = $element->innertext; + $item['uri'] = self::BASE_URI . implode('/', array_map('urlencode', explode('/', $element->href))); + + $this->items[] = $item; + } + } +} diff --git a/bridges/ContainerLinuxReleasesBridge.php b/bridges/ContainerLinuxReleasesBridge.php index ae43888..d2f6325 100644 --- a/bridges/ContainerLinuxReleasesBridge.php +++ b/bridges/ContainerLinuxReleasesBridge.php @@ -15,7 +15,6 @@ class ContainerLinuxReleasesBridge extends BridgeAbstract { 'channel' => [ 'name' => 'Release Channel', 'type' => 'list', - 'required' => true, 'defaultValue' => self::STABLE, 'values' => [ 'Stable' => self::STABLE, diff --git a/bridges/CourrierInternationalBridge.php b/bridges/CourrierInternationalBridge.php index 1e7c93e..1b754e3 100644 --- a/bridges/CourrierInternationalBridge.php +++ b/bridges/CourrierInternationalBridge.php @@ -3,7 +3,7 @@ class CourrierInternationalBridge extends BridgeAbstract { const MAINTAINER = 'teromene'; const NAME = 'Courrier International Bridge'; - const URI = 'http://CourrierInternational.com/'; + const URI = 'https://www.courrierinternational.com/'; const CACHE_TIMEOUT = 300; // 5 min const DESCRIPTION = 'Courrier International bridge'; diff --git a/bridges/CuriousCatBridge.php b/bridges/CuriousCatBridge.php new file mode 100644 index 0000000..0ebc8bd --- /dev/null +++ b/bridges/CuriousCatBridge.php @@ -0,0 +1,109 @@ + array( + 'name' => 'Username', + 'type' => 'text', + 'required' => true, + 'exampleValue' => 'koethekoethe', + ) + )); + + const CACHE_TIMEOUT = 3600; + + public function collectData() { + + $url = self::URI . '/api/v2/profile?username=' . urlencode($this->getInput('username')); + + $apiJson = getContents($url) + or returnServerError('Could not request: ' . $url); + + $apiData = json_decode($apiJson, true); + + foreach($apiData['posts'] as $post) { + $item = array(); + + $item['author'] = 'Anonymous'; + + if ($post['senderData']['id'] !== false) { + $item['author'] = $post['senderData']['username']; + } + + $item['uri'] = $this->getURI() . '/post/' . $post['id']; + $item['title'] = $this->ellipsisTitle($post['comment']); + + $item['content'] = $this->processContent($post); + $item['timestamp'] = $post['timestamp']; + + $this->items[] = $item; + } + } + + public function getURI() { + + if (!is_null($this->getInput('username'))) { + return self::URI . '/' . $this->getInput('username'); + } + + return parent::getURI(); + } + + public function getName() { + + if (!is_null($this->getInput('username'))) { + return $this->getInput('username') . ' - Curious Cat'; + } + + return parent::getName(); + } + + private function processContent($post) { + + $author = 'Anonymous'; + + if ($post['senderData']['id'] !== false) { + $authorUrl = self::URI . '/' . $post['senderData']['username']; + + $author = <<{$post['senderData']['username']} +EOD; + } + + $question = $this->formatUrls($post['comment']); + $answer = $this->formatUrls($post['reply']); + + $content = <<{$author} asked:

+
{$question}

+

{$post['addresseeData']['username']} answered:

+
{$answer}
+EOD; + + return $content; + } + + private function ellipsisTitle($text) { + $length = 150; + + if (strlen($text) > $length) { + $text = explode('
', wordwrap($text, $length, '
')); + return $text[0] . '...'; + } + + return $text; + } + + private function formatUrls($content) { + + return preg_replace( + '/(http[s]{0,1}\:\/\/[a-zA-Z0-9.\/\?\&=\-_]{4,})/ims', + '$1 ', + $content + ); + + } +} diff --git a/bridges/DailymotionBridge.php b/bridges/DailymotionBridge.php index ff8d482..dc4f5d3 100644 --- a/bridges/DailymotionBridge.php +++ b/bridges/DailymotionBridge.php @@ -4,7 +4,7 @@ class DailymotionBridge extends BridgeAbstract { const MAINTAINER = 'mitsukarenai'; const NAME = 'Dailymotion Bridge'; const URI = 'https://www.dailymotion.com/'; - const CACHE_TIMEOUT = 10800; // 3h + const CACHE_TIMEOUT = 3600; // 1h const DESCRIPTION = 'Returns the 5 newest videos by username/playlist or search'; const PARAMETERS = array ( @@ -27,74 +27,99 @@ class DailymotionBridge extends BridgeAbstract { ), 'pa' => array( 'name' => 'Page', - 'type' => 'number' + 'type' => 'number', + 'defaultValue' => 1, ) ) ); - protected function getMetadata($id){ - $metadata = array(); - $html2 = getSimpleHTMLDOM(self::URI . 'video/' . $id); - if(!$html2) { - return $metadata; - } + private $feedName = ''; - $metadata['title'] = $html2->find('meta[property=og:title]', 0)->getAttribute('content'); - $metadata['timestamp'] = strtotime( - $html2->find('meta[property=video:release_date]', 0)->getAttribute('content') - ); - $metadata['thumbnailUri'] = $html2->find('meta[property=og:image]', 0)->getAttribute('content'); - $metadata['uri'] = $html2->find('meta[property=og:url]', 0)->getAttribute('content'); - return $metadata; - } + private $apiUrl = 'https://api.dailymotion.com'; + private $apiFields = 'created_time,description,id,owner.screenname,tags,thumbnail_url,title,url'; public function getIcon() { return 'https://static1-ssl.dmcdn.net/images/neon/favicons/android-icon-36x36.png.vf806ca4ed0deed812'; } - public function collectData(){ - $html = ''; - $limit = 5; - $count = 0; + public function collectData() { + + if ($this->queriedContext === 'By username' || $this->queriedContext === 'By playlist id') { + + $apiJson = getContents($this->getApiUrl()) + or returnServerError('Could not request: ' . $this->getApiUrl()); + + $apiData = json_decode($apiJson, true); + + $this->feedName = $this->getPlaylistTitle($this->getInput('p')); + + foreach ($apiData['list'] as $apiItem) { + $item = array(); + + $item['uri'] = $apiItem['url']; + $item['uid'] = $apiItem['id']; + $item['title'] = $apiItem['title']; + $item['timestamp'] = $apiItem['created_time']; + $item['author'] = $apiItem['owner.screenname']; + $item['content'] = '

+

' . $apiItem['description'] . '

'; + $item['categories'] = $apiItem['tags']; + $item['enclosures'][] = $apiItem['thumbnail_url']; - $html = getSimpleHTMLDOM($this->getURI()) - or returnServerError('Could not request Dailymotion.'); + $this->items[] = $item; + } + } - foreach($html->find('div.media a.preview_link') as $element) { - if($count < $limit) { + if ($this->queriedContext === 'From search results') { + + $html = getSimpleHTMLDOM($this->getURI()) + or returnServerError('Could not request Dailymotion.'); + + foreach($html->find('div.media a.preview_link') as $element) { $item = array(); + $item['id'] = str_replace('/video/', '', strtok($element->href, '_')); $metadata = $this->getMetadata($item['id']); + if(empty($metadata)) { continue; } + $item['uri'] = $metadata['uri']; $item['title'] = $metadata['title']; $item['timestamp'] = $metadata['timestamp']; $item['content'] = '
' - . $item['title'] - . ''; + . $item['uri'] + . '">
' + . $item['title'] + . ''; $this->items[] = $item; - $count++; + + if (count($this->items) >= 5) { + break; + } } } } - public function getName(){ + public function getName() { switch($this->queriedContext) { case 'By username': $specific = $this->getInput('u'); break; case 'By playlist id': $specific = strtok($this->getInput('p'), '_'); + + if ($this->feedName) { + $specific = $this->feedName; + } + break; case 'From search results': $specific = $this->getInput('s'); @@ -102,26 +127,77 @@ class DailymotionBridge extends BridgeAbstract { default: return parent::getName(); } - return $specific . ' : Dailymotion Bridge'; + return $specific . ' : Dailymotion'; } public function getURI(){ $uri = self::URI; switch($this->queriedContext) { case 'By username': - $uri .= 'user/' . urlencode($this->getInput('u')) . '/1'; + $uri .= 'user/' . urlencode($this->getInput('u')); break; case 'By playlist id': $uri .= 'playlist/' . urlencode(strtok($this->getInput('p'), '_')); break; case 'From search results': $uri .= 'search/' . urlencode($this->getInput('s')); - if($this->getInput('pa')) { - $uri .= '/' . $this->getInput('pa'); + + if(!is_null($this->getInput('pa'))) { + $pa = $this->getInput('pa'); + + if ($this->getInput('pa') < 1) { + $pa = 1; + } + + $uri .= '/' . $pa; } break; default: return parent::getURI(); } return $uri; } + + private function getMetadata($id) { + $metadata = array(); + + $html = getSimpleHTMLDOM(self::URI . 'video/' . $id); + + if(!$html) { + return $metadata; + } + + $metadata['title'] = $html->find('meta[property=og:title]', 0)->getAttribute('content'); + $metadata['timestamp'] = strtotime( + $html->find('meta[property=video:release_date]', 0)->getAttribute('content') + ); + $metadata['thumbnailUri'] = $html->find('meta[property=og:image]', 0)->getAttribute('content'); + $metadata['uri'] = $html->find('meta[property=og:url]', 0)->getAttribute('content'); + return $metadata; + } + + private function getPlaylistTitle($id) { + $title = ''; + + $url = self::URI . 'playlist/' . $id; + + $html = getSimpleHTMLDOM($url) + or returnServerError('Could not request: ' . $url); + + $title = $html->find('meta[property=og:title]', 0)->getAttribute('content'); + return $title; + } + + private function getApiUrl() { + + switch($this->queriedContext) { + case 'By username': + return $this->apiUrl . '/user/' . $this->getInput('u') + . '/videos?fields=' . urlencode($this->apiFields) . '&availability=1&sort=recent&limit=5'; + break; + case 'By playlist id': + return $this->apiUrl . '/playlist/' . $this->getInput('p') + . '/videos?fields=' . urlencode($this->apiFields) . '&limit=5'; + break; + } + } } diff --git a/bridges/DanbooruBridge.php b/bridges/DanbooruBridge.php index 755399f..ea4b2be 100644 --- a/bridges/DanbooruBridge.php +++ b/bridges/DanbooruBridge.php @@ -40,7 +40,7 @@ class DanbooruBridge extends BridgeAbstract { defaultLinkTo($element, $this->getURI()); $item = array(); - $item['uri'] = $element->find('a', 0)->href; + $item['uri'] = html_entity_decode($element->find('a', 0)->href); $item['postid'] = (int)preg_replace('/[^0-9]/', '', $element->getAttribute(static::IDATTRIBUTE)); $item['timestamp'] = time(); $thumbnailUri = $element->find('img', 0)->src; diff --git a/bridges/DavesTrailerPageBridge.php b/bridges/DavesTrailerPageBridge.php new file mode 100644 index 0000000..90afec4 --- /dev/null +++ b/bridges/DavesTrailerPageBridge.php @@ -0,0 +1,27 @@ +find('tr[!align]') as $tr) { + $item = array(); + + // title + $item['title'] = $tr->find('td', 0)->find('b', 0)->plaintext; + + // content + $item['content'] = $tr->find('ul', 1); + + // uri + $item['uri'] = $tr->find('a', 3)->getAttribute('href'); + + $this->items[] = $item; + } + } +} diff --git a/bridges/DealabsBridge.php b/bridges/DealabsBridge.php index 89183ed..1657b8b 100644 --- a/bridges/DealabsBridge.php +++ b/bridges/DealabsBridge.php @@ -15,13 +15,11 @@ class DealabsBridge extends PepperBridgeAbstract { 'hide_expired' => array( 'name' => 'Masquer les éléments expirés', 'type' => 'checkbox', - 'required' => true ), 'hide_local' => array( 'name' => 'Masquer les deals locaux', 'type' => 'checkbox', 'title' => 'Masquer les deals en magasins physiques', - 'required' => true ), 'priceFrom' => array( 'name' => 'Prix minimum', @@ -41,7 +39,6 @@ class DealabsBridge extends PepperBridgeAbstract { 'group' => array( 'name' => 'Groupe', 'type' => 'list', - 'required' => true, 'title' => 'Groupe dont il faut afficher les deals', 'values' => array( 'Abonnements internet' => 'abonnements-internet', @@ -957,7 +954,6 @@ class DealabsBridge extends PepperBridgeAbstract { 'order' => array( 'name' => 'Trier par', 'type' => 'list', - 'required' => true, 'title' => 'Ordre de tri des deals', 'values' => array( 'Du deal le plus Hot au moins Hot' => '', @@ -1149,7 +1145,7 @@ class PepperBridgeAbstract extends BridgeAbstract { } else { foreach ($list as $deal) { $item = array(); - $item['uri'] = $deal->find('div[class=threadGrid-title]', 0)->find('a', 0)->href; + $item['uri'] = $deal->find('div[class*=threadGrid-title]', 0)->find('a', 0)->href; $item['title'] = $deal->find('a[class*=' . $selectorLink . ']', 0 )->plaintext; $item['author'] = $deal->find('span.thread-username', 0)->plaintext; @@ -1380,8 +1376,11 @@ class PepperBridgeAbstract extends BridgeAbstract { // Add the Hour and minutes $date_str .= ' 00:00'; - $date = DateTime::createFromFormat('j F Y H:i', $date_str); + // In some case, the date is not recognized : as a workaround the actual date is taken + if($date === false) { + $date = new DateTime(); + } return $date->getTimestamp(); } diff --git a/bridges/DemoBridge.php b/bridges/DemoBridge.php deleted file mode 100644 index f48b451..0000000 --- a/bridges/DemoBridge.php +++ /dev/null @@ -1,46 +0,0 @@ - array( - 'testCheckbox' => array( - 'type' => 'checkbox', - 'name' => 'test des checkbox' - ) - ), - 'testList' => array( - 'testList' => array( - 'type' => 'list', - 'name' => 'test des listes', - 'values' => array( - 'Test' => 'test', - 'Test 2' => 'test2' - ) - ) - ), - 'testNumber' => array( - 'testNumber' => array( - 'type' => 'number', - 'name' => 'test des numéros', - 'exampleValue' => '1515632' - ) - ) - ); - - public function collectData(){ - - $item = array(); - $item['author'] = 'Me!'; - $item['title'] = 'Test'; - $item['content'] = 'Awesome content !'; - $item['id'] = 'Lalala'; - $item['uri'] = 'http://example.com/test'; - - $this->items[] = $item; - } -} diff --git a/bridges/DemonoidBridge.php b/bridges/DemonoidBridge.php deleted file mode 100644 index 842b421..0000000 --- a/bridges/DemonoidBridge.php +++ /dev/null @@ -1,169 +0,0 @@ - array( - 'q' => array( - 'name' => 'keywords', - 'exampleValue' => 'keyword1 keyword2…', - 'required' => true, - ), - 'category' => array( - 'name' => 'Category', - 'type' => 'list', - 'values' => array( - 'All' => 0, - 'Movies' => 1, - 'Music' => 2, - 'TV' => 3, - 'Games' => 4, - 'Applications' => 5, - 'Pictures' => 8, - 'Anime' => 9, - 'Comics' => 10, - 'Books' => 11, - 'Audiobooks' => 17 - ) - ) - ), - 'Category Only' => array( - 'catOnly' => array( - 'name' => 'Category', - 'type' => 'list', - 'values' => array( - 'All' => 0, - 'Movies' => 1, - 'Music' => 2, - 'TV' => 3, - 'Games' => 4, - 'Applications' => 5, - 'Pictures' => 8, - 'Anime' => 9, - 'Comics' => 10, - 'Books' => 11, - 'Audiobooks' => 17 - ) - ) - ), - 'User ID' => array( - 'userid' => array( - 'name' => 'user id', - 'exampleValue' => '00000', - 'required' => true, - 'type' => 'number' - ), - 'category' => array( - 'name' => 'Category', - 'type' => 'list', - 'values' => array( - 'All' => 0, - 'Movies' => 1, - 'Music' => 2, - 'TV' => 3, - 'Games' => 4, - 'Applications' => 5, - 'Pictures' => 8, - 'Anime' => 9, - 'Comics' => 10, - 'Books' => 11, - 'Audiobooks' => 17 - ) - ) - ) - ); - - public function collectData() { - - if(!empty($this->getInput('q'))) { - - $html = getSimpleHTMLDOM( - self::URI . - 'files/?category=' . - rawurlencode($this->getInput('category')) . - '&subcategory=All&quality=All&seeded=2&external=2&query=' . - urlencode($this->getInput('q')) . - '&uid=0&sort=' - ) or returnServerError('Could not request Demonoid.'); - - } elseif(!empty($this->getInput('catOnly'))) { - - $html = getSimpleHTMLDOM( - self::URI . - 'files/?uid=0&category=' . - rawurlencode($this->getInput('catOnly')) . - '&subcategory=0&language=0&seeded=2&quality=0&query=&sort=' - ) or returnServerError('Could not request Demonoid.'); - - } elseif(!empty($this->getInput('userid'))) { - - $html = getSimpleHTMLDOM( - self::URI . - 'files/?uid=' . - rawurlencode($this->getInput('userid')) . - '&seeded=2' - ) or returnServerError('Could not request Demonoid.'); - - } else { - returnServerError('Invalid parameters !'); - } - - if(preg_match('~No torrents found~', $html)) { - return; - } - - $table = $html->find('td[class=ctable_content_no_pad]', 0); - $cursorCount = 4; - $elementCount = 0; - while($elementCount != 40) { - $elementCount++; - $currentElement = $table->find('tr', $cursorCount); - if(preg_match('~items total~', $currentElement)) { - break; - } - $item = array(); - //Do we have a date ? - if(preg_match('~Added.*?(.*)~', $currentElement->plaintext, $dateStr)) { - if(preg_match('~today~', $dateStr[0])) { - date_default_timezone_set('UTC'); - $timestamp = mktime(0, 0, 0, gmdate('n'), gmdate('j'), gmdate('Y')); - } else { - preg_match('~(?<=ed on ).*\d+~', $currentElement->plaintext, $fullDateStr); - date_default_timezone_set('UTC'); - $dateObj = strptime($fullDateStr[0], '%A, %b %d, %Y'); - $timestamp = mktime(0, 0, 0, $dateObj['tm_mon'] + 1, $dateObj['tm_mday'], 1900 + $dateObj['tm_year']); - } - $cursorCount++; - } - - $content = $table->find('tr', $cursorCount)->find('a', 1); - $cursorCount++; - $torrentInfo = $table->find('tr', $cursorCount); - $item['timestamp'] = $timestamp; - $item['title'] = $content->plaintext; - $item['id'] = self::URI . $content->href; - $item['uri'] = self::URI . $content->href; - $item['author'] = $torrentInfo->find('a[class=user]', 0)->plaintext; - $item['seeders'] = $torrentInfo->find('font[class=green]', 0)->plaintext; - $item['leechers'] = $torrentInfo->find('font[class=red]', 0)->plaintext; - $item['size'] = $torrentInfo->find('td', 3)->plaintext; - $item['content'] = 'Uploaded by ' . $item['author'] - . ' , Size ' . $item['size'] - . '
seeders: ' - . $item['seeders'] - . ' | leechers: ' - . $item['leechers'] - . '
info page'; - - $this->items[] = $item; - - $cursorCount++; - } - } -} diff --git a/bridges/DesoutterBridge.php b/bridges/DesoutterBridge.php index 14e26c2..0aae41a 100644 --- a/bridges/DesoutterBridge.php +++ b/bridges/DesoutterBridge.php @@ -15,7 +15,6 @@ class DesoutterBridge extends BridgeAbstract { 'news_lang' => array( 'name' => 'Language', 'type' => 'list', - 'required' => true, 'title' => 'Select your language', 'defaultValue' => 'Corporate', 'values' => array( @@ -66,7 +65,6 @@ class DesoutterBridge extends BridgeAbstract { 'industry_lang' => array( 'name' => 'Language', 'type' => 'list', - 'required' => true, 'title' => 'Select your language', 'defaultValue' => 'Corporate', 'values' => array( @@ -117,7 +115,6 @@ class DesoutterBridge extends BridgeAbstract { 'full' => array( 'name' => 'Load full articles', 'type' => 'checkbox', - 'required' => false, 'title' => 'Enable to load the full article for each item' ) ) @@ -162,13 +159,13 @@ class DesoutterBridge extends BridgeAbstract { foreach($html->find('article') as $article) { $item = array(); - $item['uri'] = $article->find('[itemprop="name"]', 0)->href; - $item['title'] = $article->find('[itemprop="name"]', 0)->title; + $item['uri'] = $article->find('a', 0)->href; + $item['title'] = $article->find('a[title]', 0)->title; if($this->getInput('full')) { $item['content'] = $this->getFullNewsArticle($item['uri']); } else { - $item['content'] = $article->find('[itemprop="description"]', 0)->plaintext; + $item['content'] = $article->find('div.tile-body p', 0)->plaintext; } $this->items[] = $item; diff --git a/bridges/DollbooruBridge.php b/bridges/DollbooruBridge.php deleted file mode 100644 index 5ed4119..0000000 --- a/bridges/DollbooruBridge.php +++ /dev/null @@ -1,9 +0,0 @@ -find('article') as $element) { + + $a = $element->find('a', 0); + $href = self::URI . $a->href; + $full = getSimpleHTMLDOMCached($href); + $article = $full->find('article', 0); + + $header = $article->find('h1', 0); + $author = $article->find('span[itemprop="author"]', 0); + $time = $article->find('time[itemprop="dateCreated"]', 0); + $content = $article->find('div[itemprop="description"]', 0); + + // Remove newsletter subscription box + $newsletter = $content->find('div[class="newsletter-form__message"]', 0); + if ($newsletter) + $newsletter->outertext = ''; + + $newsletterForm = $content->find('form', 0); + if ($newsletterForm) + $newsletterForm->outertext = ''; + + // Remove next and previous article URLs at the bottom + $nextprev = $content->find('div[class="blog-post__next-previous-wrapper"]', 0); + if ($nextprev) + $nextprev->outertext = ''; + + $section = [ $article->find('h3[itemprop="articleSection"]', 0)->plaintext ]; + + $item = array(); + $item['title'] = $header->find('span', 0)->innertext . ': ' + . $header->find('span', 1)->innertext; + + $item['uri'] = $href; + $item['timestamp'] = strtotime($time->datetime); + $item['author'] = $author->innertext; + $item['categories'] = $section; + + $item['content'] = '' . $content->innertext; + + $this->items[] = $item; + + if (count($this->items) >= 10) + break; + } + } +} diff --git a/bridges/EliteDangerousGalnetBridge.php b/bridges/EliteDangerousGalnetBridge.php index dc6077b..1afa042 100644 --- a/bridges/EliteDangerousGalnetBridge.php +++ b/bridges/EliteDangerousGalnetBridge.php @@ -47,5 +47,8 @@ class EliteDangerousGalnetBridge extends BridgeAbstract { $this->items[] = $item; } + + //Remove duplicates that sometimes show up on the website + $this->items = array_unique($this->items, SORT_REGULAR); } } diff --git a/bridges/ElloBridge.php b/bridges/ElloBridge.php index 22f5d30..3de167e 100644 --- a/bridges/ElloBridge.php +++ b/bridges/ElloBridge.php @@ -120,9 +120,11 @@ class ElloBridge extends BridgeAbstract { } private function getAPIKey() { - $cache = Cache::create('FileCache'); - $cache->setPath(PATH_CACHE); - $cache->setParameters(['key']); + $cacheFac = new CacheFactory(); + $cacheFac->setWorkingDir(PATH_LIB_CACHES); + $cache = $cacheFac->create(Configuration::getConfig('cache', 'type')); + $cache->setScope(get_called_class()); + $cache->setKey(['key']); $key = $cache->loadData(); if($key == null) { diff --git a/bridges/EngadgetBridge.php b/bridges/EngadgetBridge.php new file mode 100644 index 0000000..cf200fa --- /dev/null +++ b/bridges/EngadgetBridge.php @@ -0,0 +1,26 @@ +collectExpandableDatas(static::URI . 'rss.xml', 15); + } + + protected function parseItem($newsItem){ + $item = parent::parseItem($newsItem); + // $articlePage gets the entire page's contents + $articlePage = getSimpleHTMLDOM($newsItem->link); + // figure contain's the main article image + $article = $articlePage->find('figure', 0); + // .article-text has the actual article + foreach($articlePage->find('.article-text') as $element) + $article = $article . $element; + $item['content'] = $article; + return $item; + } +} diff --git a/bridges/ExtremeDownloadBridge.php b/bridges/ExtremeDownloadBridge.php index 5272997..acdf630 100644 --- a/bridges/ExtremeDownloadBridge.php +++ b/bridges/ExtremeDownloadBridge.php @@ -15,7 +15,6 @@ class ExtremeDownloadBridge extends BridgeAbstract { 'filter' => array( 'name' => 'Type de contenu', 'type' => 'list', - 'required' => true, 'title' => 'Type de contenu à suivre : Téléchargement, Streaming ou les deux', 'values' => array( 'Streaming et Téléchargement' => 'both', diff --git a/bridges/FB2Bridge.php b/bridges/FB2Bridge.php index 29df755..2faa321 100644 --- a/bridges/FB2Bridge.php +++ b/bridges/FB2Bridge.php @@ -72,15 +72,15 @@ class FB2Bridge extends BridgeAbstract { $pageInfo = $this->getPageInfos($page, $cookies); if($pageInfo['userId'] === null) { - echo <<find('article') as $content) { $item = array(); - //echo $content; die(); + preg_match('/publish_time\\\":([0-9]+),/', $content->getAttribute('data-store', 0), $match); if(isset($match[1])) $timestamp = $match[1]; diff --git a/bridges/FDroidBridge.php b/bridges/FDroidBridge.php index b606cec..7f54735 100644 --- a/bridges/FDroidBridge.php +++ b/bridges/FDroidBridge.php @@ -11,7 +11,6 @@ class FDroidBridge extends BridgeAbstract { 'u' => array( 'name' => 'Widget selection', 'type' => 'list', - 'required' => true, 'values' => array( 'Latest added apps' => 'added', 'Latest updated apps' => 'updated' @@ -29,14 +28,14 @@ class FDroidBridge extends BridgeAbstract { or returnServerError('Could not request F-Droid.'); // targetting the corresponding widget based on user selection - // "updated" is the 4th widget on the page, "added" is the 5th + // "updated" is the 5th widget on the page, "added" is the 6th switch($this->getInput('u')) { case 'updated': - $html_widget = $html->find('div.sidebar-widget', 4); + $html_widget = $html->find('div.sidebar-widget', 5); break; default: - $html_widget = $html->find('div.sidebar-widget', 5); + $html_widget = $html->find('div.sidebar-widget', 6); break; } diff --git a/bridges/FabriceBellardBridge.php b/bridges/FabriceBellardBridge.php new file mode 100644 index 0000000..2c24b5e --- /dev/null +++ b/bridges/FabriceBellardBridge.php @@ -0,0 +1,36 @@ +find('p') as $obj) { + $item = array(); + + $html = defaultLinkTo($html, $this->getURI()); + + $links = $obj->find('a'); + if (count($links) > 0) { + $link_uri = $links[0]->href; + } else { + $link_uri = $this->getURI(); + } + + /* try to make sure the link is valid */ + if ($link_uri[-1] !== '/' && strpos($link_uri, '/') === false) { + $link_uri = $link_uri . '/'; + } + + $item['title'] = strip_tags($obj->innertext); + $item['uri'] = $link_uri; + $item['content'] = $obj->innertext; + + $this->items[] = $item; + } + } +} diff --git a/bridges/FacebookBridge.php b/bridges/FacebookBridge.php index 7b61705..08b3a38 100644 --- a/bridges/FacebookBridge.php +++ b/bridges/FacebookBridge.php @@ -142,7 +142,11 @@ class FacebookBridge extends BridgeAbstract { private function collectGroupData() { - $header = array('Accept-Language: ' . getEnv('HTTP_ACCEPT_LANGUAGE') . "\r\n"); + if(getEnv('HTTP_ACCEPT_LANGUAGE')) { + $header = array('Accept-Language: ' . getEnv('HTTP_ACCEPT_LANGUAGE')); + } else { + $header = array(); + } $html = getSimpleHTMLDOM($this->getURI(), $header) or returnServerError('Failed loading facebook page: ' . $this->getURI()); @@ -219,8 +223,7 @@ class FacebookBridge extends BridgeAbstract { $ogtitle = $html->find('meta[property="og:title"]', 0) or returnServerError('Unable to find group title!'); - return htmlspecialchars_decode($ogtitle->content, ENT_QUOTES); - + return html_entity_decode($ogtitle->content, ENT_QUOTES); } private function extractGroupURI($post) { @@ -506,7 +509,11 @@ EOD; // Retrieve page contents if(is_null($html)) { - $header = array('Accept-Language: ' . getEnv('HTTP_ACCEPT_LANGUAGE')); + if(getEnv('HTTP_ACCEPT_LANGUAGE')) { + $header = array('Accept-Language: ' . getEnv('HTTP_ACCEPT_LANGUAGE')); + } else { + $header = array(); + } $html = getSimpleHTMLDOM($this->getURI(), $header) or returnServerError('No results for this query.'); @@ -581,6 +588,8 @@ EOD; '._5mly', // Remove embedded videos (the preview image remains) '._2ezg', // Remove "Views ..." '.hidden_elem', // Remove hidden elements (they are hidden anyway) + '.timestampContent', // Remove relative timestamp + '._6spk', // Remove redundant separator ); foreach($content_filters as $filter) { diff --git a/bridges/FeedExpanderExampleBridge.php b/bridges/FeedExpanderExampleBridge.php deleted file mode 100644 index 537a635..0000000 --- a/bridges/FeedExpanderExampleBridge.php +++ /dev/null @@ -1,62 +0,0 @@ - array( - 'version' => array( - 'name' => 'Version', - 'type' => 'list', - 'required' => true, - 'title' => 'Select your feed format/version', - 'defaultValue' => 'RSS 2.0', - 'values' => array( - 'RSS 0.91' => 'rss_0_9_1', - 'RSS 1.0' => 'rss_1_0', - 'RSS 2.0' => 'rss_2_0', - 'ATOM 1.0' => 'atom_1_0' - ) - ) - ) - ); - - public function collectData(){ - switch($this->getInput('version')) { - case 'rss_0_9_1': - parent::collectExpandableDatas('http://static.userland.com/gems/backend/sampleRss.xml'); - break; - case 'rss_1_0': - parent::collectExpandableDatas('http://feeds.nature.com/nature/rss/current?format=xml'); - break; - case 'rss_2_0': - parent::collectExpandableDatas('http://feeds.rssboard.org/rssboard?format=xml'); - break; - case 'atom_1_0': - parent::collectExpandableDatas('http://segfault.linuxmint.com/feed/atom/'); - break; - default: returnClientError('Unknown version ' . $this->getInput('version') . '!'); - } - } - - protected function parseItem($newsItem) { - switch($this->getInput('version')) { - case 'rss_0_9_1': - return $this->parseRSS_0_9_1_Item($newsItem); - break; - case 'rss_1_0': - return $this->parseRSS_1_0_Item($newsItem); - break; - case 'rss_2_0': - return $this->parseRSS_2_0_Item($newsItem); - break; - case 'atom_1_0': - return $this->parseATOMItem($newsItem); - break; - default: returnClientError('Unknown version ' . $this->getInput('version') . '!'); - } - } -} diff --git a/bridges/FicbookBridge.php b/bridges/FicbookBridge.php new file mode 100644 index 0000000..8b8a57f --- /dev/null +++ b/bridges/FicbookBridge.php @@ -0,0 +1,164 @@ + array(), + 'Fiction Updates' => array( + 'fiction_id' => array( + 'name' => 'Fanfiction ID', + 'type' => 'text', + 'pattern' => '[0-9]+', + 'required' => true, + 'title' => 'Insert fanfiction ID', + 'exampleValue' => '5783919', + ), + 'include_contents' => array( + 'name' => 'Include contents', + 'type' => 'checkbox', + 'title' => 'Activate to include contents in the feed', + ), + ), + 'Fiction Comments' => array( + 'fiction_id' => array( + 'name' => 'Fanfiction ID', + 'type' => 'text', + 'pattern' => '[0-9]+', + 'required' => true, + 'title' => 'Insert fanfiction ID', + 'exampleValue' => '5783919', + ), + ), + ); + + public function getURI() { + switch($this->queriedContext) { + case 'Site News': { + // For some reason this is not HTTPS + return 'http://ficbook.net/sitenews'; + } + case 'Fiction Updates': { + return self::URI + . 'readfic/' + . urlencode($this->getInput('fiction_id')); + } + case 'Fiction Comments': { + return self::URI + . 'readfic/' + . urlencode($this->getInput('fiction_id')) + . '/comments#content'; + } + default: return parent::getURI(); + } + } + + public function collectData() { + + $header = array('Accept-Language: en-US'); + + $html = getSimpleHTMLDOM($this->getURI(), $header) + or returnServerError('Could not request ' . $this->getURI()); + + $html = defaultLinkTo($html, self::URI); + + switch($this->queriedContext) { + case 'Site News': return $this->collectSiteNews($html); + case 'Fiction Updates': return $this->collectUpdatesData($html); + case 'Fiction Comments': return $this->collectCommentsData($html); + } + + } + + private function collectSiteNews($html) { + foreach($html->find('.news_view') as $news) { + $this->items[] = array( + 'title' => $news->find('h1.title', 0)->plaintext, + 'timestamp' => strtotime($this->fixDate($news->find('span[title]', 0)->title)), + 'content' => $news->find('.news_text', 0), + ); + } + } + + private function collectCommentsData($html) { + foreach($html->find('article.post') as $article) { + $this->items[] = array( + 'uri' => $article->find('.comment_link_to_fic > a', 0)->href, + 'title' => $article->find('.comment_author', 0)->plaintext, + 'author' => $article->find('.comment_author', 0)->plaintext, + 'timestamp' => strtotime($this->fixDate($article->find('time[datetime]', 0)->datetime)), + 'content' => $article->find('.comment_message', 0), + 'enclosures' => array($article->find('img', 0)->src), + ); + } + } + + private function collectUpdatesData($html) { + foreach($html->find('ul.table-of-contents > li') as $chapter) { + $item = array( + 'uri' => $chapter->find('a', 0)->href, + 'title' => $chapter->find('a', 0)->plaintext, + 'timestamp' => strtotime($this->fixDate($chapter->find('span[title]', 0)->title)), + ); + + if($this->getInput('include_contents')) { + $content = getSimpleHTMLDOMCached($item['uri']); + $item['content'] = $content->find('#content', 0); + } + + $this->items[] = $item; + + // Sort by time, descending + usort($this->items, function($a, $b){ return $b['timestamp'] - $a['timestamp']; }); + } + } + + private function fixDate($date) { + + // FIXME: This list was generated using Google tranlator. Someone who + // actually knows russian should check this list! Please keep in mind + // that month names must match exactly the names returned by Ficbook. + $ru_month = array( + 'января', + 'февраля', + 'марта', + 'апреля', + 'мая', + 'июня', + 'июля', + 'августа', + 'Сентября', + 'октября', + 'Ноября', + 'Декабря', + ); + + $en_month = array( + 'January', + 'February', + 'March', + 'April', + 'May', + 'June', + 'July', + 'August', + 'September', + 'October', + 'November', + 'December', + ); + + $fixed_date = str_replace($ru_month, $en_month, $date); + + if($fixed_date === $date) { + Debug::log('Unable to fix date: ' . $date); + return null; + } + + return $fixed_date; + + } +} diff --git a/bridges/FindACrewBridge.php b/bridges/FindACrewBridge.php index c245c84..abab6e1 100644 --- a/bridges/FindACrewBridge.php +++ b/bridges/FindACrewBridge.php @@ -62,11 +62,16 @@ class FindACrewBridge extends BridgeAbstract { foreach ($annonces as $annonce) { $item = array(); - $img = parent::getURI() . $annonce->find('.css_LstPic img', 0)->getAttribute('src'); - $item['title'] = $annonce->find('.css_LstCtrls span', 0)->plaintext; - $item['uri'] = parent::getURI() . $annonce->find('.css_PnlCtrls a', 0)->href; - $content = $annonce->find('.css_LstDtl div', 2)->innertext; - $item['content'] = "
$content"; + $link = parent::getURI() . $annonce->find('.lst-ctrls a', 0)->href; + $htmlDetail = getSimpleHTMLDOMCached($link . '?mdl=2'); // add ?mdl=2 for xhr content not full html page + + $img = parent::getURI() . $htmlDetail->find('img.img-responsive', 0)->getAttribute('src'); + $item['title'] = $annonce->find('.lst-tags span', 0)->plaintext; + $item['uri'] = $link; + $content = $htmlDetail->find('.panel-body div.clearfix.row > div', 1)->innertext; + $content .= $htmlDetail->find('.panel-body > div', 1)->innertext; + $content = defaultLinkTo($content, parent::getURI()); + $item['content'] = $content; $item['enclosures'] = array($img); $item['categories'] = array($annonce->find('.css_AccLocCur', 0)->plaintext); $this->items[] = $item; diff --git a/bridges/FurAffinityBridge.php b/bridges/FurAffinityBridge.php new file mode 100644 index 0000000..2f78ee4 --- /dev/null +++ b/bridges/FurAffinityBridge.php @@ -0,0 +1,918 @@ + array( + 'q' => array( + 'name' => 'Query', + 'required' => true + ), + 'rating-general' => array( + 'name' => 'General', + 'type' => 'checkbox', + 'defaultValue' => 'checked' + ), + 'rating-mature' => array( + 'name' => 'Mature', + 'type' => 'checkbox', + ), + 'rating-adult' => array( + 'name' => 'Adult', + 'type' => 'checkbox', + ), + 'range' => array( + 'name' => 'Time range', + 'type' => 'list', + 'values' => array( + 'A Day' => 'day', + '3 Days' => '3days', + 'A Week' => 'week', + 'A Month' => 'month', + 'All time' => 'all' + ), + 'defaultValue' => 'all' + ), + 'type-art' => array( + 'name' => 'Art', + 'type' => 'checkbox', + 'defaultValue' => 'checked' + ), + 'type-flash' => array( + 'name' => 'Flash', + 'type' => 'checkbox', + 'defaultValue' => 'checked' + ), + 'type-photo' => array( + 'name' => 'Photography', + 'type' => 'checkbox', + 'defaultValue' => 'checked' + ), + 'type-music' => array( + 'name' => 'Music', + 'type' => 'checkbox', + 'defaultValue' => 'checked' + ), + 'type-story' => array( + 'name' => 'Story', + 'type' => 'checkbox', + 'defaultValue' => 'checked' + ), + 'type-poetry' => array( + 'name' => 'Poetry', + 'type' => 'checkbox', + 'defaultValue' => 'checked' + ), + 'mode' => array( + 'name' => 'Match mode', + 'type' => 'list', + 'values' => array( + 'All of the words' => 'all', + 'Any of the words' => 'any', + 'Extended' => 'extended' + ), + 'defaultValue' => 'extended' + ), + 'limit' => array( + 'name' => 'Limit', + 'type' => 'number', + 'defaultValue' => 10, + 'title' => 'Limit number of submissions to return. -1 for unlimited.' + ), + 'full' => array( + 'name' => 'Full view', + 'title' => 'Include description, tags, date and larger image in article. Uses more bandwidth.', + 'type' => 'checkbox', + 'defaultValue' => 'checked' + ), + 'cache' => array( + 'name' => 'Cache submission pages', + 'title' => 'Reduces requests to FA when Full view is enabled. Changes to submission details may be delayed.', + 'type' => 'checkbox', + 'defaultValue' => 'checked' + ) + ), + 'Browse' => array( + 'cat' => array( + 'name' => 'Category', + 'type' => 'list', + 'values' => array( + 'Visual Art' => array( + 'All' => 1, + 'Artwork (Digital)' => 2, + 'Artwork (Traditional)' => 3, + 'Cellshading' => 4, + 'Crafting' => 5, + 'Designs' => 6, + 'Flash' => 7, + 'Fursuiting' => 8, + 'Icons' => 9, + 'Mosaics' => 10, + 'Photography' => 11, + 'Sculpting' => 12 + ), + 'Readable Art' => array( + 'Story' => 13, + 'Poetry' => 14, + 'Prose' => 15 + ), + 'Audio Art' => array( + 'Music' => 16, + 'Podcasts' => 17 + ), + 'Downloadable' => array( + 'Skins' => 18, + 'Handhelds' => 19, + 'Resources' => 20 + ), + 'Other Stuff' => array( + 'Adoptables' => 21, + 'Auctions' => 22, + 'Contests' => 23, + 'Current Events' => 24, + 'Desktops' => 25, + 'Stockart' => 26, + 'Screenshots' => 27, + 'Scraps' => 28, + 'Wallpaper' => 29, + 'YCH / Sale' => 30, + 'Other' => 31 + ) + ), + 'defaultValue' => 1 + ), + 'atype' => array( + 'name' => 'Type', + 'type' => 'list', + 'values' => array( + 'General Things' => array( + 'All' => 1, + 'Abstract' => 2, + 'Animal related (non-anthro)' => 3, + 'Anime' => 4, + 'Comics' => 5, + 'Doodle' => 6, + 'Fanart' => 7, + 'Fantasy' => 8, + 'Human' => 9, + 'Portraits' => 10, + 'Scenery' => 11, + 'Still Life' => 12, + 'Tutorials' => 13, + 'Miscellaneous' => 14 + ), + 'Fetish / Furry specialty' => array( + 'Baby fur' => 101, + 'Bondage' => 102, + 'Digimon' => 103, + 'Fat Furs' => 104, + 'Fetish Other' => 105, + 'Fursuit' => 106, + 'Gore / Macabre Art' => 119, + 'Hyper' => 107, + 'Inflation' => 108, + 'Macro / Micro' => 109, + 'Muscle' => 110, + 'My Little Pony / Brony' => 111, + 'Paw' => 112, + 'Pokemon' => 113, + 'Pregnancy' => 114, + 'Sonic' => 115, + 'Transformation' => 116, + 'Vore' => 117, + 'Water Sports' => 118, + 'General Furry Art' => 100 + ), + 'Music' => array( + 'Techno' => 201, + 'Trance' => 202, + 'House' => 203, + '90s' => 204, + '80s' => 205, + '70s' => 206, + '60s' => 207, + 'Pre-60s' => 208, + 'Classical' => 209, + 'Game Music' => 210, + 'Rock' => 211, + 'Pop' => 212, + 'Rap' => 213, + 'Industrial' => 214, + 'Other Music' => 200 + ) + ), + 'defaultValue' => 1 + ), + 'species' => array( + 'name' => 'Species', + 'type' => 'list', + 'values' => array( + 'Unspecified / Any' => 1, + 'Amphibian' => array( + 'Frog' => 1001, + 'Newt' => 1002, + 'Salamander' => 1003, + 'Amphibian (Other)' => 1000 + ), + 'Aquatic' => array( + 'Cephalopod' => 2001, + 'Dolphin' => 2002, + 'Fish' => 2005, + 'Porpoise' => 2004, + 'Seal' => 6068, + 'Shark' => 2006, + 'Whale' => 2003, + 'Aquatic (Other)' => 2000 + ), + 'Avian' => array( + 'Corvid' => 3001, + 'Crow' => 3002, + 'Duck' => 3003, + 'Eagle' => 3004, + 'Falcon' => 3005, + 'Goose' => 3006, + 'Gryphon' => 3007, + 'Hawk' => 3008, + 'Owl' => 3009, + 'Phoenix' => 3010, + 'Swan' => 3011, + 'Avian (Other)' => 3000 + ), + 'Bears & Ursines' => array( + 'Bear' => 6002 + ), + 'Camelids' => array( + 'Camel' => 6074, + 'Llama' => 6036 + ), + 'Canines & Lupines' => array( + 'Coyote' => 6008, + 'Doberman' => 6009, + 'Dog' => 6010, + 'Dingo' => 6011, + 'German Shepherd' => 6012, + 'Jackal' => 6013, + 'Husky' => 6014, + 'Wolf' => 6016, + 'Canine (Other)' => 6017 + ), + 'Cervines' => array( + 'Cervine (Other)' => 6018 + ), + 'Cows & Bovines' => array( + 'Antelope' => 6004, + 'Cows' => 6003, + 'Gazelle' => 6005, + 'Goat' => 6006, + 'Bovines (General)' => 6007 + ), + 'Dragons' => array( + 'Eastern Dragon' => 4001, + 'Hydra' => 4002, + 'Serpent' => 4003, + 'Western Dragon' => 4004, + 'Wyvern' => 4005, + 'Dragon (Other)' => 4000 + ), + 'Equestrians' => array( + 'Donkey' => 6019, + 'Horse' => 6034, + 'Pony' => 6073, + 'Zebra' => 6071 + ), + 'Exotic & Mythicals' => array( + 'Argonian' => 5002, + 'Chakat' => 5003, + 'Chocobo' => 5004, + 'Citra' => 5005, + 'Crux' => 5006, + 'Daemon' => 5007, + 'Digimon' => 5008, + 'Dracat' => 5009, + 'Draenei' => 5010, + 'Elf' => 5011, + 'Gargoyle' => 5012, + 'Iksar' => 5013, + 'Kaiju/Monster' => 5015, + 'Langurhali' => 5014, + 'Moogle' => 5017, + 'Naga' => 5016, + 'Orc' => 5018, + 'Pokemon' => 5019, + 'Satyr' => 5020, + 'Sergal' => 5021, + 'Tanuki' => 5022, + 'Unicorn' => 5023, + 'Xenomorph' => 5024, + 'Alien (Other)' => 5001, + 'Exotic (Other)' => 5000 + ), + 'Felines' => array( + 'Domestic Cat' => 6020, + 'Cheetah' => 6021, + 'Cougar' => 6022, + 'Jaguar' => 6023, + 'Leopard' => 6024, + 'Lion' => 6025, + 'Lynx' => 6026, + 'Ocelot' => 6027, + 'Panther' => 6028, + 'Tiger' => 6029, + 'Feline (Other)' => 6030 + ), + 'Insects' => array( + 'Arachnid' => 8000, + 'Mantid' => 8004, + 'Scorpion' => 8005, + 'Insect (Other)' => 8003 + ), + 'Mammals (Other)' => array( + 'Bat' => 6001, + 'Giraffe' => 6031, + 'Hedgehog' => 6032, + 'Hippopotamus' => 6033, + 'Hyena' => 6035, + 'Panda' => 6052, + 'Pig/Swine' => 6053, + 'Rabbit/Hare' => 6059, + 'Raccoon' => 6060, + 'Red Panda' => 6062, + 'Meerkat' => 6043, + 'Mongoose' => 6044, + 'Rhinoceros' => 6063, + 'Mammals (Other)' => 6000 + ), + 'Marsupials' => array( + 'Opossum' => 6037, + 'Kangaroo' => 6038, + 'Koala' => 6039, + 'Quoll' => 6040, + 'Wallaby' => 6041, + 'Marsupial (Other)' => 6042 + ), + 'Mustelids' => array( + 'Badger' => 6045, + 'Ferret' => 6046, + 'Mink' => 6048, + 'Otter' => 6047, + 'Skunk' => 6069, + 'Weasel' => 6049, + 'Mustelid (Other)' => 6051 + ), + 'Primates' => array( + 'Gorilla' => 6054, + 'Human' => 6055, + 'Lemur' => 6056, + 'Monkey' => 6057, + 'Primate (Other)' => 6058 + ), + 'Reptillian' => array( + 'Alligator & Crocodile' => 7001, + 'Gecko' => 7003, + 'Iguana' => 7004, + 'Lizard' => 7005, + 'Snakes & Serpents' => 7006, + 'Turtle' => 7007, + 'Reptilian (Other)' => 7000 + ), + 'Rodents' => array( + 'Beaver' => 6064, + 'Mouse' => 6065, + 'Rat' => 6061, + 'Squirrel' => 6070, + 'Rodent (Other)' => 6067 + ), + 'Vulpines' => array( + 'Fennec' => 6072, + 'Fox' => 6075, + 'Vulpine (Other)' => 6015 + ), + 'Other' => array( + 'Dinosaur' => 8001, + 'Wolverine' => 6050 + ) + ), + 'defaultValue' => 1 + ), + 'gender' => array( + 'name' => 'Gender', + 'type' => 'list', + 'values' => array( + 'Any' => 0, + 'Male' => 2, + 'Female' => 3, + 'Herm' => 4, + 'Transgender' => 5, + 'Multiple characters' => 6, + 'Other / Not Specified' => 7 + ), + 'defaultValue' => 0 + ), + 'rating_general' => array( + 'name' => 'General', + 'type' => 'checkbox', + 'defaultValue' => 'checked' + ), + 'rating_mature' => array( + 'name' => 'Mature', + 'type' => 'checkbox', + ), + 'rating_adult' => array( + 'name' => 'Adult', + 'type' => 'checkbox', + ), + 'limit-browse' => array( + 'name' => 'Limit', + 'type' => 'number', + 'required' => true, + 'defaultValue' => 10, + 'title' => 'Limit number of submissions to return. -1 for unlimited.' + ), + 'full' => array( + 'name' => 'Full view', + 'title' => 'Include description, tags, date and larger image in article. Uses more bandwidth.', + 'type' => 'checkbox', + 'defaultValue' => 'checked' + ), + 'cache' => array( + 'name' => 'Cache submission pages', + 'title' => 'Reduces requests to FA when Full view is enabled. Changes to submission details may be delayed.', + 'type' => 'checkbox', + 'defaultValue' => 'checked' + ) + + ), + 'Journals' => array( + 'username-journals' => array( + 'name' => 'Username', + 'required' => true, + 'title' => 'Lowercase username as seen in URLs' + ), + 'limit' => array( + 'name' => 'Limit', + 'type' => 'number', + 'defaultValue' => -1, + 'title' => 'Limit number of journals to return. -1 for unlimited.' + ) + + ), + 'Single Journal' => array( + 'journal-id' => array( + 'name' => 'Journal ID', + 'required' => true, + 'type' => 'number', + 'title' => 'Number seen in journal URL' + ) + ), + 'Gallery' => array( + 'username-gallery' => array( + 'name' => 'Username', + 'required' => true, + 'title' => 'Lowercase username as seen in URLs' + ), + 'limit' => array( + 'name' => 'Limit', + 'type' => 'number', + 'defaultValue' => 10, + 'title' => 'Limit number of submissions to return. -1 for unlimited.' + ), + 'full' => array( + 'name' => 'Full view', + 'title' => 'Include description, tags, date and larger image in article. Uses more bandwidth.', + 'type' => 'checkbox', + 'defaultValue' => 'checked' + ), + 'cache' => array( + 'name' => 'Cache submission pages', + 'title' => 'Reduces requests to FA when Full view is enabled. Changes to submission details may be delayed.', + 'type' => 'checkbox', + 'defaultValue' => 'checked' + ) + ), + 'Scraps' => array( + 'username-scraps' => array( + 'name' => 'Username', + 'required' => true, + 'title' => 'Lowercase username as seen in URLs' + ), + 'limit' => array( + 'name' => 'Limit', + 'type' => 'number', + 'defaultValue' => 10, + 'title' => 'Limit number of submissions to return. -1 for unlimited.' + ), + 'full' => array( + 'name' => 'Full view', + 'title' => 'Include description, tags, date and larger image in article. Uses more bandwidth.', + 'type' => 'checkbox', + 'defaultValue' => 'checked' + ), + 'cache' => array( + 'name' => 'Cache submission pages', + 'title' => 'Reduces requests to FA when Full view is enabled. Changes to submission details may be delayed.', + 'type' => 'checkbox', + 'defaultValue' => 'checked' + ) + ), + 'Favorites' => array( + 'username-favorites' => array( + 'name' => 'Username', + 'required' => true, + 'title' => 'Lowercase username as seen in URLs' + ), + 'limit' => array( + 'name' => 'Limit', + 'type' => 'number', + 'defaultValue' => 10, + 'title' => 'Limit number of submissions to return. -1 for unlimited.' + ), + 'full' => array( + 'name' => 'Full view', + 'title' => 'Include description, tags, date and larger image in article. Uses more bandwidth.', + 'type' => 'checkbox', + 'defaultValue' => 'checked' + ), + 'cache' => array( + 'name' => 'Cache submission pages', + 'title' => 'Reduces requests to FA when Full view is enabled. Changes to submission details may be delayed.', + 'type' => 'checkbox', + 'defaultValue' => 'checked' + ) + ), + 'Gallery Folder' => array( + 'username-folder' => array( + 'name' => 'Username', + 'required' => true, + 'title' => 'Lowercase username as seen in URLs' + ), + 'folder-id' => array( + 'name' => 'Folder ID', + 'required' => true, + 'type' => 'number', + 'title' => 'Number seen in folder URL' + ), + 'limit' => array( + 'name' => 'Limit', + 'type' => 'number', + 'defaultValue' => 10, + 'title' => 'Limit number of submissions to return. -1 for unlimited.' + ), + 'full' => array( + 'name' => 'Full view', + 'title' => 'Include description, tags, date and larger image in article. Uses more bandwidth.', + 'type' => 'checkbox', + 'defaultValue' => 'checked' + ), + 'cache' => array( + 'name' => 'Cache submission pages', + 'title' => 'Reduces requests to FA when Full view is enabled. Changes to submission details may be delayed.', + 'type' => 'checkbox', + 'defaultValue' => 'checked' + ) + ) + ); + + /* + * This was aquired by creating a new user on FA then + * extracting the cookie from the browsers dev console. + */ + const FA_AUTH_COOKIE = 'b=4ce65691-b50f-4742-a990-bf28d6de16ee; a=ca6e4566-9d81-4263-9444-653b142e35f8'; + + public function detectParameters($url) { + $params = array(); + + // Single journal + $regex = '/^(https?:\/\/)?(www\.)?furaffinity.net\/journal\/(\d+)/'; + if(preg_match($regex, $url, $matches) > 0) { + $params['journal-id'] = urldecode($matches[3]); + return $params; + } + + // Journals + $regex = '/^(https?:\/\/)?(www\.)?furaffinity.net\/journals\/([^\/&?\n]+)/'; + if(preg_match($regex, $url, $matches) > 0) { + $params['username-journals'] = urldecode($matches[3]); + return $params; + } + + // Gallery folder + $regex = '/^(https?:\/\/)?(www\.)?furaffinity.net\/gallery\/([^\/&?\n]+)\/folder\/(\d+)/'; + if(preg_match($regex, $url, $matches) > 0) { + $params['username-folder'] = urldecode($matches[3]); + $params['folder-id'] = urldecode($matches[4]); + $params['full'] = 'on'; + return $params; + } + + // Gallery (must be after gallery folder) + $regex = '/^(https?:\/\/)?(www\.)?furaffinity.net\/(gallery|scraps|favorites)\/([^\/&?\n]+)/'; + if(preg_match($regex, $url, $matches) > 0) { + $params['username-' . $matches[3]] = urldecode($matches[4]); + $params['full'] = 'on'; + return $params; + } + + return null; + } + + public function getName() { + switch($this->queriedContext) { + case 'Search': + return 'Search For ' + . $this->getInput('q'); + case 'Browse': + return 'Browse'; + case 'Journals': + return $this->getInput('username-journals'); + case 'Single Journal': + return 'Journal ' + . $this->getInput('journal-id'); + case 'Gallery': + return $this->getInput('username-gallery'); + case 'Scraps': + return $this->getInput('username-scraps'); + case 'Favorites': + return $this->getInput('username-favorites'); + case 'Gallery Folder': + return $this->getInput('username-folder') + . '\'s Folder ' + . $this->getInput('folder-id'); + default: return parent::getName(); + } + } + + public function getDescription() { + switch($this->queriedContext) { + case 'Search': + return 'FurAffinity Search For ' + . $this->getInput('q'); + case 'Browse': + return 'FurAffinity Browse'; + case 'Journals': + return 'FurAffinity Journals By ' + . $this->getInput('username-journals'); + case 'Single Journal': + return 'FurAffinity Journal ' + . $this->getInput('journal-id'); + case 'Gallery': + return 'FurAffinity Gallery By ' + . $this->getInput('username-gallery'); + case 'Scraps': + return 'FurAffinity Scraps By ' + . $this->getInput('username-scraps'); + case 'Favorites': + return 'FurAffinity Favorites By ' + . $this->getInput('username-favorites'); + case 'Gallery Folder': + return 'FurAffinity Gallery Folder ' + . $this->getInput('folder-id') + . ' By ' + . $this->getInput('username-folder'); + default: return parent::getDescription(); + } + } + + public function getURI() { + switch($this->queriedContext) { + case 'Search': + return SELF::URI + . '/search'; + case 'Browse': + return SELF::URI + . '/browse'; + case 'Journals': + return SELF::URI + . '/journals/' + . $this->getInput('username-journals'); + case 'Single Journal': + return SELF::URI + . '/journal/' + . $this->getInput('journal-id'); + case 'Gallery': + return SELF::URI + . '/gallery/' + . $this->getInput('username-gallery'); + case 'Scraps': + return SELF::URI + . '/scraps/' + . $this->getInput('username-scraps'); + case 'Favorites': + return SELF::URI + . '/favorites/' + . $this->getInput('username-favorites'); + case 'Gallery Folder': + return SELF::URI + . '/gallery/' + . $this->getInput('username-folder') + . '/folder/' + . $this->getInput('folder-id'); + default: return parent::getURI(); + } + } + + public function collectData() { + switch($this->queriedContext) { + case 'Search': + $data = array( + 'q' => $this->getInput('q'), + 'perpage' => 72, + 'rating-general' => ($this->getInput('rating-general') === true ? 'on' : 0), + 'rating-mature' => ($this->getInput('rating-mature') === true ? 'on' : 0), + 'rating-adult' => ($this->getInput('rating-adult') === true ? 'on' : 0), + 'range' => $this->getInput('range'), + 'type-art' => ($this->getInput('type-art') === true ? 'on' : 0), + 'type-flash' => ($this->getInput('type-flash') === true ? 'on' : 0), + 'type-photo' => ($this->getInput('type-photo') === true ? 'on' : 0), + 'type-music' => ($this->getInput('type-music') === true ? 'on' : 0), + 'type-story' => ($this->getInput('type-story') === true ? 'on' : 0), + 'type-poetry' => ($this->getInput('type-poetry') === true ? 'on' : 0), + 'mode' => $this->getInput('mode') + ); + $html = $this->postFASimpleHTMLDOM($data); + $limit = (is_int($this->getInput('limit')) ? $this->getInput('limit') : 10); + $this->itemsFromSubmissionList($html, $limit); + break; + case 'Browse': + $data = array( + 'cat' => $this->getInput('cat'), + 'atype' => $this->getInput('atype'), + 'species' => $this->getInput('species'), + 'gender' => $this->getInput('gender'), + 'perpage' => 72, + 'rating_general' => ($this->getInput('rating_general') === true ? 'on' : 0), + 'rating_mature' => ($this->getInput('rating_mature') === true ? 'on' : 0), + 'rating_adult' => ($this->getInput('rating_adult') === true ? 'on' : 0) + ); + $html = $this->postFASimpleHTMLDOM($data); + $limit = (is_int($this->getInput('limit-browse')) ? $this->getInput('limit-browse') : 10); + $this->itemsFromSubmissionList($html, $limit); + break; + case 'Journals': + $html = $this->getFASimpleHTMLDOM($this->getURI()); + $limit = (is_int($this->getInput('limit')) ? $this->getInput('limit') : -1); + $this->itemsFromJournalList($html, $limit); + break; + case 'Single Journal': + $html = $this->getFASimpleHTMLDOM($this->getURI()); + $this->itemsFromJournal($html); + break; + case 'Gallery': + case 'Scraps': + case 'Favorites': + case 'Gallery Folder': + $html = $this->getFASimpleHTMLDOM($this->getURI()); + $limit = (is_int($this->getInput('limit')) ? $this->getInput('limit') : 10); + $this->itemsFromSubmissionList($html, $limit); + break; + } + } + + private function postFASimpleHTMLDOM($data) { + $opts = array( + CURLOPT_CUSTOMREQUEST => 'POST', + CURLOPT_POSTFIELDS => http_build_query($data) + ); + $header = array( + 'Host: ' . parse_url(self::URI, PHP_URL_HOST), + 'Content-Type: application/x-www-form-urlencoded', + 'Cookie: ' . self::FA_AUTH_COOKIE + ); + + $html = getSimpleHTMLDOM($this->getURI(), $header, $opts); + $html = defaultLinkTo($html, $this->getURI()); + + return $html; + } + + private function getFASimpleHTMLDOM($url, $cache = false) { + $header = array( + 'Cookie: ' . self::FA_AUTH_COOKIE + ); + + if($cache) { + $html = getSimpleHTMLDOMCached($url, 86400, $header); // 24 hours + } else { + $html = getSimpleHTMLDOM($url, $header); + } + + $html = defaultLinkTo($html, $url); + + return $html; + } + + private function itemsFromJournalList($html, $limit) { + foreach($html->find('table[id^=jid:]') as $journal) { + # allows limit = -1 to mean 'unlimited' + if($limit-- === 0) break; + + $item = array(); + + $this->setReferrerPolicy($journal); + + $item['uri'] = $journal->find('a', 0)->href; + $item['title'] = html_entity_decode($journal->find('a', 0)->plaintext); + $item['author'] = $this->getInput('username-journals'); + $item['timestamp'] = strtotime( + $journal->find('span.popup_date', 0)->plaintext); + $item['content'] = $journal + ->find('.alt1 table div.no_overflow', 0) + ->innertext; + + $this->items[] = $item; + } + } + + private function itemsFromJournal($html) { + $this->setReferrerPolicy($html); + $item = array(); + + $item['uri'] = $this->getURI(); + + $title = $html->find('.journal-title-box .no_overflow', 0)->plaintext; + $title = html_entity_decode($title); + $title = trim($title, " \t\n\r\0\x0B" . chr(0xC2) . chr(0xA0)); + $item['title'] = $title; + + $item['author'] = $html->find('.journal-title-box a', 0)->plaintext; + $item['timestamp'] = strtotime( + $html->find('.journal-title-box span.popup_date', 0)->plaintext); + $item['content'] = $html->find('.journal-body', 0)->innertext; + + $this->items[] = $item; + } + + private function itemsFromSubmissionList($html, $limit) { + $cache = ($this->getInput('cache') === true); + + foreach($html->find('section.gallery figure') as $figure) { + # allows limit = -1 to mean 'unlimited' + if($limit-- === 0) break; + + $item = array(); + + $submissionURL = $figure->find('b u a', 0)->href; + $imgURL = 'https:' . $figure->find('b u a img', 0)->src; + + $item['uri'] = $submissionURL; + $item['title'] = html_entity_decode( + $figure->find('figcaption p a[href*=/view/]', 0)->title); + $item['author'] = $figure->find('figcaption p a[href*=/user/]', 0)->title; + + if($this->getInput('full') === true) { + $submissionHTML = $this->getFASimpleHTMLDOM($submissionURL, $cache); + + $stats = $submissionHTML->find('.stats-container', 0); + $item['timestamp'] = strtotime($stats->find('.popup_date', 0)->title); + $item['enclosures'] = array( + $submissionHTML->find('.actions a[href^=https://d.facdn]', 0)->href + ); + foreach($stats->find('#keywords a') as $keyword) { + $item['categories'][] = $keyword->plaintext; + } + + $previewSrc = $submissionHTML->find('#submissionImg', 0) + ->{'data-preview-src'}; + if($previewSrc) { + $imgURL = 'https:' . $previewSrc; + } + + $description = $submissionHTML + ->find('.maintable .maintable tr td.alt1', -1); + $this->setReferrerPolicy($description); + $description = $description->innertext; + + $item['content'] = << + + +

+{$description} +

+EOD; + } else { + $item['content'] = << + + +EOD; + } + + $this->items[] = $item; + } + } + + private function setReferrerPolicy(&$html) { + foreach($html->find('img') as $img) { + /* + * Note: Without the no-referrer policy their CDN sometimes denies requests. + * We can't control this for enclosures sadly. + * At least tt-rss adds the referrerpolicy on its own. + * Alternatively we could not use https for images, but that's not ideal. + */ + $img->referrerpolicy = 'no-referrer'; + } + } +} diff --git a/bridges/GBAtempBridge.php b/bridges/GBAtempBridge.php index 9383be7..48a7f85 100644 --- a/bridges/GBAtempBridge.php +++ b/bridges/GBAtempBridge.php @@ -10,7 +10,6 @@ class GBAtempBridge extends BridgeAbstract { 'type' => array( 'name' => 'Type', 'type' => 'list', - 'required' => true, 'values' => array( 'News' => 'N', 'Reviews' => 'R', diff --git a/bridges/GOGBridge.php b/bridges/GOGBridge.php index 669332f..09f47b4 100644 --- a/bridges/GOGBridge.php +++ b/bridges/GOGBridge.php @@ -8,8 +8,8 @@ class GOGBridge extends BridgeAbstract { public function collectData() { - $values = getContents('https://www.gog.com/games/ajax/filtered?limit=25&sort=new') or - die('Unable to get the news pages from GOG !'); + $values = getContents('https://www.gog.com/games/ajax/filtered?limit=25&sort=new') + or returnServerError('Unable to get the news pages from GOG !'); $decodedValues = json_decode($values); $limit = 0; @@ -38,8 +38,8 @@ class GOGBridge extends BridgeAbstract { private function buildGameContentPage($game) { - $gameDescriptionText = getContents('https://api.gog.com/products/' . $game->id . '?expand=description') or - die('Unable to get game description from GOG !'); + $gameDescriptionText = getContents('https://api.gog.com/products/' . $game->id . '?expand=description') + or returnServerError('Unable to get game description from GOG !'); $gameDescriptionValue = json_decode($gameDescriptionText); diff --git a/bridges/GQMagazineBridge.php b/bridges/GQMagazineBridge.php index 961b3a0..2884ab6 100644 --- a/bridges/GQMagazineBridge.php +++ b/bridges/GQMagazineBridge.php @@ -40,6 +40,11 @@ class GQMagazineBridge extends BridgeAbstract 'data-original' => 'src' ); + const POSSIBLE_TITLES = array( + 'h2', + 'h3' + ); + private function getDomain() { $domain = $this->getInput('domain'); if (empty($domain)) @@ -54,6 +59,17 @@ class GQMagazineBridge extends BridgeAbstract return $this->getDomain() . '/' . $this->getInput('page'); } + private function findTitleOf($link) { + foreach (self::POSSIBLE_TITLES as $tag) { + $title = $link->parent()->find($tag, 0); + if($title !== null) { + if($title->plaintext !== null) { + return $title->plaintext; + } + } + } + } + public function collectData() { $html = getSimpleHTMLDOM($this->getURI()) or returnServerError('Could not request ' . $this->getURI()); @@ -61,31 +77,36 @@ class GQMagazineBridge extends BridgeAbstract // Since GQ don't want simple class scrapping, let's do it the hard way and ... discover content ! $main = $html->find('main', 0); foreach ($main->find('a') as $link) { + if(strpos($link, $this->getInput('page'))) + continue; $uri = $link->href; - $title = $link->find('h2', 0); - $date = $link->find('time', 0); + $date = $link->parent()->find('time', 0); $item = array(); - $author = $link->find('span[itemprop=name]', 0); - $item['author'] = $author->plaintext; - $item['title'] = $title->plaintext; - if(substr($uri, 0, 1) === 'h') { // absolute uri - $item['uri'] = $uri; - } else if(substr($uri, 0, 1) === '/') { // domain relative url - $item['uri'] = $this->getDomain() . $uri; - } else { - $item['uri'] = $this->getDomain() . '/' . $uri; - } - - $article = $this->loadFullArticle($item['uri']); - if($article) { - $item['content'] = $this->replaceUriInHtmlElement($article); - } else { - $item['content'] = "Article body couldn't be loaded. It must be a bug!"; + $author = $link->parent()->find('span[itemprop=name]', 0); + if($author !== null) { + $item['author'] = $author->plaintext; + $item['title'] = $this->findTitleOf($link); + switch(substr($uri, 0, 1)) { + case 'h': // absolute uri + $item['uri'] = $uri; + break; + case '/': // domain relative uri + $item['uri'] = $this->getDomain() . $uri; + break; + default: + $item['uri'] = $this->getDomain() . '/' . $uri; + } + $article = $this->loadFullArticle($item['uri']); + if($article) { + $item['content'] = $this->replaceUriInHtmlElement($article); + } else { + $item['content'] = "Article body couldn't be loaded. It must be a bug!"; + } + $short_date = $date->datetime; + $item['timestamp'] = strtotime($short_date); + $this->items[] = $item; } - $short_date = $date->datetime; - $item['timestamp'] = strtotime($short_date); - $this->items[] = $item; } } @@ -96,16 +117,7 @@ class GQMagazineBridge extends BridgeAbstract */ private function loadFullArticle($uri){ $html = getSimpleHTMLDOMCached($uri); - // Once again, that generated css classes madness is an obstacle ... which i can go over easily - foreach($html->find('div') as $div) { - // List the CSS classes of that div - $classes = $div->class; - // I can't directly lookup that class since GQ since to generate random names like "ArticleBodySection-fkggUW" - if(strpos($classes, 'ArticleBodySection') !== false) { - return $div; - } - } - return null; + return $html->find('section[data-test-id=ArticleBodyContent]', 0); } /** diff --git a/bridges/GiteaBridge.php b/bridges/GiteaBridge.php new file mode 100644 index 0000000..3324787 --- /dev/null +++ b/bridges/GiteaBridge.php @@ -0,0 +1,27 @@ +find('#release-list > li') + or returnServerError('Unable to find releases'); + + foreach($releases as $release) { + $this->items[] = array( + 'uri' => $release->find('a', 0)->href, + 'title' => 'Release ' . $release->find('h3', 0)->plaintext, + ); + } + } +} diff --git a/bridges/GithubIssueBridge.php b/bridges/GithubIssueBridge.php index 91dd45e..2eddeb2 100644 --- a/bridges/GithubIssueBridge.php +++ b/bridges/GithubIssueBridge.php @@ -66,10 +66,21 @@ class GithubIssueBridge extends BridgeAbstract { return parent::getURI(); } - protected function extractIssueEvent($issueNbr, $title, $comment){ - $comment = $comment->firstChild(); - $uri = static::URI . $this->getInput('u') . '/' . $this->getInput('p') - . '/issues/' . $issueNbr . '#' . $comment->getAttribute('id'); + private function buildGitHubIssueCommentUri($issue_number, $comment_id) { + // https://github.com///issues/# + return static::URI + . $this->getInput('u') + . '/' + . $this->getInput('p') + . '/issues/' + . $issue_number + . '#' + . $comment_id; + } + + private function extractIssueEvent($issueNbr, $title, $comment){ + + $uri = $this->buildGitHubIssueCommentUri($issueNbr, $comment->id); $author = $comment->find('.author', 0)->plaintext; @@ -94,22 +105,21 @@ class GithubIssueBridge extends BridgeAbstract { return $item; } - protected function extractIssueComment($issueNbr, $title, $comment){ - $uri = static::URI . $this->getInput('u') . '/' - . $this->getInput('p') . '/issues/' . $issueNbr; + private function extractIssueComment($issueNbr, $title, $comment){ + + $uri = $this->buildGitHubIssueCommentUri($issueNbr, $comment->parent->id); $author = $comment->find('.author', 0)->plaintext; $title .= ' / ' . trim( - $comment->find('.comment .timeline-comment-header-text', 0)->plaintext + $comment->find('.timeline-comment-header-text', 0)->plaintext ); $content = $comment->find('.comment-body', 0)->innertext; $item = array(); $item['author'] = $author; - $item['uri'] = $uri - . '#' . $comment->firstChild()->nextSibling()->getAttribute('id'); + $item['uri'] = $uri; $item['title'] = html_entity_decode($title, ENT_QUOTES, 'UTF-8'); $item['timestamp'] = strtotime( $comment->find('relative-time', 0)->getAttribute('datetime') @@ -118,25 +128,32 @@ class GithubIssueBridge extends BridgeAbstract { return $item; } - protected function extractIssueComments($issue){ + private function extractIssueComments($issue){ $items = array(); $title = $issue->find('.gh-header-title', 0)->plaintext; $issueNbr = trim( substr($issue->find('.gh-header-number', 0)->plaintext, 1) ); - $comments = $issue->find('.js-discussion', 0); - foreach($comments->children() as $comment) { + + $comments = $issue->find(' + [id^="issue-"] > .comment, + [id^="issuecomment-"] > .comment, + [id^="event-"], + [id^="ref-"] + '); + foreach($comments as $comment) { + if (!$comment->hasChildNodes()) { continue; } - $comment = $comment->firstChild(); - $classes = explode(' ', $comment->getAttribute('class')); - if (in_array('timeline-comment-wrapper', $classes)) { + + if (!$comment->hasClass('discussion-item-header')) { $item = $this->extractIssueComment($issueNbr, $title, $comment); $items[] = $item; continue; } - while (in_array('discussion-item', $classes)) { + + while ($comment->hasClass('discussion-item-header')) { $item = $this->extractIssueEvent($issueNbr, $title, $comment); $items[] = $item; $comment = $comment->nextSibling(); @@ -145,6 +162,7 @@ class GithubIssueBridge extends BridgeAbstract { } $classes = explode(' ', $comment->getAttribute('class')); } + } return $items; } @@ -192,8 +210,13 @@ class GithubIssueBridge extends BridgeAbstract { ENT_QUOTES, 'UTF-8' ); - $comments = trim($issue->find('.col-5', 0)->plaintext); - $item['content'] .= "\n" . 'Comments: ' . ($comments ? $comments : '0'); + + $comment_count = 0; + if($span = $issue->find('a[aria-label*="comment"] span', 0)) { + $comment_count = $span->plaintext; + } + + $item['content'] .= "\n" . 'Comments: ' . $comment_count; $item['uri'] = self::URI . $issue->find('.js-navigation-open', 0)->getAttribute('href'); $this->items[] = $item; @@ -216,4 +239,43 @@ class GithubIssueBridge extends BridgeAbstract { $item['title'] = preg_replace('/\s+/', ' ', $item['title']); }); } + + public function detectParameters($url) { + + if(filter_var($url, FILTER_VALIDATE_URL, FILTER_FLAG_PATH_REQUIRED) === false + || strpos($url, self::URI) !== 0) { + return null; + } + + $url_components = parse_url($url); + $path_segments = array_values(array_filter(explode('/', $url_components['path']))); + + switch(count($path_segments)) { + case 2: { // Project issues + list($user, $project) = $path_segments; + $show_comments = 'off'; + } break; + case 3: { // Project issues with issue comments + if($path_segments[2] !== 'issues') { + return null; + } + list($user, $project) = $path_segments; + $show_comments = 'on'; + } break; + case 4: { // Issue comments + list($user, $project, /* issues */, $issue) = $path_segments; + } break; + default: { + return null; + } + } + + return array( + 'u' => $user, + 'p' => $project, + 'c' => isset($show_comments) ? $show_comments : null, + 'i' => isset($issue) ? $issue : null, + ); + + } } diff --git a/bridges/GithubSearchBridge.php b/bridges/GithubSearchBridge.php index fe8a721..fd90934 100644 --- a/bridges/GithubSearchBridge.php +++ b/bridges/GithubSearchBridge.php @@ -24,7 +24,7 @@ class GithubSearchBridge extends BridgeAbstract { $html = getSimpleHTMLDOM($url) or returnServerError('Error while downloading the website content'); - foreach($html->find('div.repo-list-item') as $element) { + foreach($html->find('li.repo-list-item') as $element) { $item = array(); $uri = $element->find('h3 a', 0)->href; diff --git a/bridges/GlassdoorBridge.php b/bridges/GlassdoorBridge.php index 1e20077..308859d 100644 --- a/bridges/GlassdoorBridge.php +++ b/bridges/GlassdoorBridge.php @@ -117,7 +117,7 @@ class GlassdoorBridge extends BridgeAbstract { $item['title'] = $post->find('header', 0)->plaintext; $item['content'] = $post->find('div[class="excerpt-content"]', 0)->plaintext; $item['enclosures'] = array( - $this->getFullSizeImageURI($post->find('div[class="post-thumb"]', 0)->{'data-original'}) + $this->getFullSizeImageURI($post->find('div[class*="post-thumb"]', 0)->{'data-original'}) ); // optionally load full articles @@ -141,7 +141,7 @@ class GlassdoorBridge extends BridgeAbstract { } private function collectReviewData($html, $limit) { - $reviews = $html->find('#EmployerReviews li[id^="empReview]') + $reviews = $html->find('#ReviewsFeed li[id^="empReview]') or returnServerError('Unable to find reviews!'); foreach($reviews as $review) { @@ -153,7 +153,19 @@ class GlassdoorBridge extends BridgeAbstract { $item['timestamp'] = strtotime($review->find('time', 0)->datetime); $mainText = $review->find('p.mainText', 0)->plaintext; - $description = $review->find('div.prosConsAdvice', 0)->innertext; + + $description = ''; + foreach($review->find('div.description p') as $p) { + + if ($p->hasClass('strong')) { + $p->tag = 'strong'; + $p->removeClass('strong'); + } + + $description .= $p; + + } + $item['content'] = "

{$mainText}

{$description}

"; $this->items[] = $item; diff --git a/bridges/GlowficBridge.php b/bridges/GlowficBridge.php new file mode 100644 index 0000000..e8975a7 --- /dev/null +++ b/bridges/GlowficBridge.php @@ -0,0 +1,88 @@ + array(), + 'Thread' => array( + 'post_id' => array( + 'name' => 'Post ID', + 'title' => 'https://www.glowfic.com/posts/', + 'type' => 'number' + ), + 'start_page' => array( + 'name' => 'Start Page', + 'title' => 'To start from an offset page', + 'type' => 'number' + ) + ) + ); + + public function collectData() { + $url = $this->getAPIURI(); + $metadata = get_headers( $url . '/replies', true ) or returnClientError('Post did not return reply headers.'); + $metadata['Last-Page'] = ceil( $metadata['Total'] / $metadata['Per-Page'] ); + if(!is_null($this->getInput('start_page')) && + $this->getInput('start_page') < 1 && $metadata['Last-Page'] - $this->getInput('start_page') > 0) { + $first_page = $metadata['Last-Page'] - $this->getInput('start_page'); + } else if(!is_null($this->getInput('start_page')) && $this->getInput('start_page') <= $metadata['Last-Page']) { + $first_page = $this->getInput('start_page'); + } else { + $first_page = 1; + } + for ($page_offset = $first_page; $page_offset <= $metadata['Last-Page']; $page_offset++) { + $jsonContents = getContents($url . '/replies?page=' . $page_offset ) or + returnClientError('Could not retrieve replies for page ' . $page_offset . '.'); + $replies = json_decode($jsonContents); + foreach ($replies as $reply) { + $item = array(); + + $item['content'] = $reply->{'content'}; + $item['uri'] = $this->getURI() . '?page=' . $page_offset . '#reply-' . $reply->{'id'}; + if ($reply->{'icon'}) { + $item['enclosures'] = array($reply->{'icon'}->{'url'}); + } + $item['author'] = $reply->{'character'}->{'screenname'} . ' (' . $reply->{'character'}->{'name'} . ')'; + $item['timestamp'] = date('r', strtotime($reply->{'created_at'})); + $item['title'] = 'Tag by ' . $reply->{'user'}->{'username'} . ' updated at ' . $reply->{'updated_at'}; + $this->items[] = $item; + } + } + } + + private function getAPIURI() { + $url = parent::getURI() . '/api/v1/posts/' . $this->getInput('post_id'); + return $url; + } + + public function getURI() { + $url = parent::getURI() . '/posts/' . $this->getInput('post_id'); + return $url; + } + + private function getPost() { + $url = $this->getAPIURI(); + $jsonPost = getContents( $url ) or returnClientError('Could not retrieve post metadata.'); + $post = json_decode($jsonPost); + return $post; + } + + public function getName(){ + if(!is_null($this->getInput('post_id'))) { + $post = $this->getPost(); + return $post->{'subject'} . ' - ' . parent::getName(); + } + return parent::getName(); + } + + public function getDescription(){ + if(!is_null($this->getInput('post_id'))) { + $post = $this->getPost(); + return $post->{'content'}; + } + return parent::getName(); + } +} diff --git a/bridges/GogsBridge.php b/bridges/GogsBridge.php new file mode 100644 index 0000000..a08bcc0 --- /dev/null +++ b/bridges/GogsBridge.php @@ -0,0 +1,206 @@ + array( + 'host' => array( + 'name' => 'Host', + 'exampleValue' => 'https://gogs.io', + 'required' => true, + 'title' => 'Host name without trailing slash', + ), + 'user' => array( + 'name' => 'Username', + 'exampleValue' => 'gogs', + 'required' => true, + 'title' => 'User name as it appears in the URL', + ), + 'project' => array( + 'name' => 'Project name', + 'exampleValue' => 'gogs', + 'required' => true, + 'title' => 'Project name as it appears in the URL', + ), + ), + 'Commits' => array( + 'branch' => array( + 'name' => 'Branch name', + 'defaultValue' => 'master', + 'required' => true, + 'title' => 'Branch name as it appears in the URL', + ), + ), + 'Issues' => array( + 'include_description' => array( + 'name' => 'Include issue description', + 'type' => 'checkbox', + 'title' => 'Activate to include the issue description', + ), + ), + 'Single issue' => array( + 'issue' => array( + 'name' => 'Issue number', + 'type' => 'number', + 'exampleValue' => 102, + 'required' => true, + 'title' => 'Issue number from the issues list', + ), + ), + 'Releases' => array(), + ); + + private $title = ''; + + /** + * Note: detectParamters doesn't make sense for this bridge because there is + * no "single" host for this service. Anyone can host it. + */ + + public function getURI() { + switch($this->queriedContext) { + case 'Commits': { + return $this->getInput('host') + . '/' . $this->getInput('user') + . '/' . $this->getInput('project') + . '/commits/' . $this->getInput('branch'); + } break; + case 'Issues': { + return $this->getInput('host') + . '/' . $this->getInput('user') + . '/' . $this->getInput('project') + . '/issues/'; + } break; + case 'Single issue': { + return $this->getInput('host') + . '/' . $this->getInput('user') + . '/' . $this->getInput('project') + . '/issues/' . $this->getInput('issue'); + } break; + case 'Releases': { + return $this->getInput('host') + . '/' . $this->getInput('user') + . '/' . $this->getInput('project') + . '/releases/'; + } break; + default: return parent::getURI(); + } + } + + public function getName() { + switch($this->queriedContext) { + case 'Commits': + case 'Issues': + case 'Releases': return $this->title . ' ' . $this->queriedContext; + case 'Single issue': return $this->title . ' Issue ' . $this->getInput('issue'); + default: return parent::getName(); + } + } + + public function getIcon() { + return 'https://gogs.io/img/favicon.ico'; + } + + public function collectData() { + + $html = getSimpleHTMLDOM($this->getURI()) + or returnServerError('Could not request ' . $this->getURI()); + + $html = defaultLinkTo($html, $this->getURI()); + + $this->title = $html->find('[property="og:title"]', 0)->content; + + switch($this->queriedContext) { + case 'Commits': { + $this->collectCommitsData($html); + } break; + case 'Issues': { + $this->collectIssuesData($html); + } break; + case 'Single issue': { + $this->collectSingleIssueData($html); + } break; + case 'Releases': { + $this->collectReleasesData($html); + } break; + } + + } + + protected function collectCommitsData($html) { + $commits = $html->find('#commits-table tbody tr') + or returnServerError('Unable to find commits'); + + foreach($commits as $commit) { + $this->items[] = array( + 'uri' => $commit->find('a.sha', 0)->href, + 'title' => $commit->find('.message span', 0)->plaintext, + 'author' => $commit->find('.author', 0)->plaintext, + 'timestamp' => $commit->find('.time-since', 0)->title, + 'uid' => $commit->find('.sha', 0)->plaintext, + ); + } + } + + protected function collectIssuesData($html) { + $issues = $html->find('.issue.list li') + or returnServerError('Unable to find issues'); + + foreach($issues as $issue) { + $uri = $issue->find('a', 0)->href; + + $item = array( + 'uri' => $uri, + 'title' => $issue->find('.label', 0)->plaintext . ' | ' . $issue->find('a.title', 0)->plaintext, + 'author' => $issue->find('.desc a', 0)->plaintext, + 'timestamp' => $issue->find('.time-since', 0)->title, + 'uid' => $issue->find('.label', 0)->plaintext, + ); + + if($this->getInput('include_description')) { + $issue_html = getSimpleHTMLDOMCached($uri, 3600) + or returnServerError('Unable to load issue description'); + + $issue_html = defaultLinkTo($issue_html, $uri); + + $item['content'] = $issue_html->find('.comment .markdown', 0); + } + + $this->items[] = $item; + } + } + + protected function collectSingleIssueData($html) { + $comments = $html->find('.comments .comment') + or returnServerError('Unable to find comments'); + + foreach($comments as $comment) { + $this->items[] = array( + 'uri' => $comment->find('a[href*="#issue"]', 0)->href, + 'title' => $comment->find('span', 0)->plaintext, + 'author' => $comment->find('.content a', 0)->plaintext, + 'timestamp' => $comment->find('.time-since', 0)->title, + 'content' => $comment->find('.markdown', 0), + ); + } + + $this->items = array_reverse($this->items); + } + + protected function collectReleasesData($html) { + $releases = $html->find('#release-list li') + or returnServerError('Unable to find releases'); + + foreach($releases as $release) { + $this->items[] = array( + 'uri' => $release->find('a', 0)->href, + 'title' => 'Release ' . $release->find('h4', 0)->plaintext, + ); + } + } +} diff --git a/bridges/GooglePlusPostBridge.php b/bridges/GooglePlusPostBridge.php deleted file mode 100644 index 7911eaf..0000000 --- a/bridges/GooglePlusPostBridge.php +++ /dev/null @@ -1,208 +0,0 @@ - array( - 'name' => 'username or Id', - 'required' => true - ), - 'include_media' => array( - 'name' => 'Include media', - 'type' => 'checkbox', - 'title' => 'Enable to include media in the feed content' - ) - )); - - public function getIcon() { - return 'https://ssl.gstatic.com/images/branding/product/ico/google_plus_alldp.ico'; - } - - public function collectData(){ - - $username = $this->getInput('username'); - - // Usernames start with a + if it's not an ID - if(!is_numeric($username) && substr($username, 0, 1) !== '+') { - $username = '+' . $username; - } - - $html = getSimpleHTMLDOM(static::URI . '/' . urlencode($username) . '/posts') - or returnServerError('No results for this query.'); - - $html = defaultLinkTo($html, static::URI); - - $this->title = $html->find('meta[property=og:title]', 0)->getAttribute('content'); - $this->url = $html->find('meta[property=og:url]', 0)->getAttribute('content'); - - foreach($html->find('div[jsname=WsjYwc]') as $post) { - - $item = array(); - - $item['author'] = $post->find('div div div div a', 0)->innertext; - $item['uri'] = $post->find('div div div a', 1)->href; - - $timestamp = $post->find('a.qXj2He span', 0); - - if($timestamp) { - $item['timestamp'] = strtotime('+' . preg_replace( - '/[^0-9A-Za-z]/', - '', - $timestamp->getAttribute('aria-label'))); - } - - $message = $post->find('div[jsname=EjRJtf]', 0); - - // Empty messages are not supported right now - if(!$message) { - continue; - } - - $item['content'] = '
' - . $item['author'] - . '
' - . trim(strip_tags($message, '

')) - . '
'; - - // Make title at least 50 characters long, but don't add '...' if it is shorter! - if(strlen($message->plaintext) > 50) { - $end = strpos($message->plaintext, ' ', 50) ?: strlen($message->plaintext); - } else { - $end = strlen($message->plaintext); - } - - if(strlen(substr($message->plaintext, 0, $end)) === strlen($message->plaintext)) { - $item['title'] = $message->plaintext; - } else { - $item['title'] = substr($message->plaintext, 0, $end) . '...'; - } - - $media = $post->find('[jsname="MTOxpb"]', 0); - - if($media) { - - $item['enclosures'] = array(); - - foreach($media->find('img') as $img) { - $item['enclosures'][] = $this->fixImage($img)->src; - } - - if($this->getInput('include_media') === true && count($item['enclosures'] > 0)) { - $item['content'] .= '
'; - } - - } - - // Add custom parameters (only useful for JSON or Plaintext) - $item['fullname'] = $item['author']; - $item['avatar'] = $post->find('div img', 0)->src; - $item['id'] = $post->find('div div div', 0)->getAttribute('id'); - $item['content_simple'] = $message->plaintext; - - $this->items[] = $item; - - } - - } - - public function getName(){ - return $this->title ?: 'Google Plus Post Bridge'; - } - - public function getURI(){ - return $this->url ?: parent::getURI(); - } - - private function fixImage($img) { - - // There are certain images like .gif which link to a static picture and - // get replaced dynamically via JS in the browser. If we want the "real" - // image we need to account for that. - - $urlparts = parse_url($img->src); - - if(array_key_exists('host', $urlparts)) { - - // For some reason some URIs don't contain the scheme, assume https - if(!array_key_exists('scheme', $urlparts)) { - $urlparts['scheme'] = 'https'; - } - - $pathelements = explode('/', $urlparts['path']); - - switch($urlparts['host']) { - - case 'lh3.googleusercontent.com': - - if(pathinfo(end($pathelements), PATHINFO_EXTENSION)) { - - // The second to last element of the path specifies the - // image format. The URL is still valid if we remove it. - unset($pathelements[count($pathelements) - 2]); - - } elseif(strrpos(end($pathelements), '=') !== false) { - - // Some images go throug a proxy. For those images they - // add size information after an equal sign. - // Example: '=w530-h298-n'. Again this can safely be - // removed to get the original image. - $pathelements[count($pathelements) - 1] = substr( - end($pathelements), - 0, - strrpos(end($pathelements), '=') - ); - - } - - break; - - } - - $urlparts['path'] = implode('/', $pathelements); - - } - - $img->src = $this->build_url($urlparts); - return $img; - - } - - /** - * From: https://gist.github.com/Ellrion/f51ba0d40ae1d62eeae44fd1adf7b704 - * slightly adjusted to work with PHP < 7.0 - * @param array $parts - * @return string - */ - private function build_url(array $parts) - { - - $scheme = isset($parts['scheme']) ? ($parts['scheme'] . '://') : ''; - $host = isset($parts['host']) ? $parts['host'] : ''; - $port = isset($parts['port']) ? (':' . $parts['port']) : ''; - $user = isset($parts['user']) ? $parts['user'] : ''; - $pass = isset($parts['pass']) ? (':' . $parts['pass']) : ''; - $pass = ($user || $pass) ? ($pass . '@') : ''; - $path = isset($parts['path']) ? $parts['path'] : ''; - $query = isset($parts['query']) ? ('?' . $parts['query']) : ''; - $fragment = isset($parts['fragment']) ? ('#' . $parts['fragment']) : ''; - - return implode('', [$scheme, $user, $pass, $host, $port, $path, $query, $fragment]); - - } -} diff --git a/bridges/HDWallpapersBridge.php b/bridges/HDWallpapersBridge.php index cea6e34..f1579e0 100644 --- a/bridges/HDWallpapersBridge.php +++ b/bridges/HDWallpapersBridge.php @@ -16,13 +16,13 @@ class HDWallpapersBridge extends BridgeAbstract { ), 'r' => array( 'name' => 'resolution', - 'defaultValue' => '1920x1200', - 'exampleValue' => '1920x1200, 1680x1050,…' + 'defaultValue' => 'HD', + 'exampleValue' => 'HD, 1920x1200, 1680x1050,…' ) )); public function collectData(){ - $category = $this->category; + $category = $this->getInput('c'); if(strrpos($category, 'wallpapers') !== strlen($category) - strlen('wallpapers')) { $category .= '-desktop-wallpapers'; } @@ -45,13 +45,12 @@ class HDWallpapersBridge extends BridgeAbstract { $thumbnail = $element->find('img', 0); $item = array(); - // http://www.hdwallpapers.in/download/yosemite_reflections-1680x1050.jpg $item['uri'] = self::URI . '/download' . str_replace('wallpapers.html', $this->getInput('r') . '.jpg', $element->href); $item['timestamp'] = time(); - $item['title'] = $element->find('p', 0)->text(); + $item['title'] = $element->find('em1', 0)->text(); $item['content'] = $item['title'] . '
'; + $item['enclosures'] = array($item['uri']); $this->items[] = $item; $num++; diff --git a/bridges/HaveIBeenPwnedBridge.php b/bridges/HaveIBeenPwnedBridge.php new file mode 100644 index 0000000..96dc7b2 --- /dev/null +++ b/bridges/HaveIBeenPwnedBridge.php @@ -0,0 +1,138 @@ + array( + 'name' => 'Order by', + 'type' => 'list', + 'values' => array( + 'Breach date' => 'breachDate', + 'Date added to HIBP' => 'dateAdded', + ), + 'defaultValue' => 'dateAdded', + ), + 'item_limit' => array( + 'name' => 'Limit number of returned items', + 'type' => 'number', + 'defaultValue' => 20, + ) + )); + + const CACHE_TIMEOUT = 3600; + + private $breachDateRegex = '/Breach date: ([0-9]{1,2} [A-Z-a-z]+ [0-9]{4})/'; + private $dateAddedRegex = '/Date added to HIBP: ([0-9]{1,2} [A-Z-a-z]+ [0-9]{4})/'; + private $accountsRegex = '/Compromised accounts: ([0-9,]+)/'; + + private $breaches = array(); + + public function collectData() { + + $html = getSimpleHTMLDOM(self::URI . '/PwnedWebsites') + or returnServerError('Could not request: ' . self::URI . '/PwnedWebsites'); + + $breaches = array(); + + foreach($html->find('div.row') as $breach) { + $item = array(); + + if ($breach->class != 'row') { + continue; + } + + preg_match($this->breachDateRegex, $breach->find('p', 1)->plaintext, $breachDate) + or returnServerError('Could not extract details'); + + preg_match($this->dateAddedRegex, $breach->find('p', 1)->plaintext, $dateAdded) + or returnServerError('Could not extract details'); + + preg_match($this->accountsRegex, $breach->find('p', 1)->plaintext, $accounts) + or returnServerError('Could not extract details'); + + $permalink = $breach->find('p', 1)->find('a', 0)->href; + + // Remove permalink + $breach->find('p', 1)->find('a', 0)->outertext = ''; + + $item['title'] = html_entity_decode($breach->find('h3', 0)->plaintext, ENT_QUOTES) + . ' - ' . $accounts[1] . ' breached accounts'; + $item['dateAdded'] = strtotime($dateAdded[1]); + $item['breachDate'] = strtotime($breachDate[1]); + $item['uri'] = self::URI . '/PwnedWebsites' . $permalink; + + $item['content'] = '

' . $breach->find('p', 0)->innertext . '

'; + $item['content'] .= '

' . $this->breachType($breach) . '

'; + $item['content'] .= '

' . $breach->find('p', 1)->innertext . '

'; + + $this->breaches[] = $item; + } + + $this->orderBreaches(); + $this->createItems(); + } + + /** + * Extract data breach type(s) + */ + private function breachType($breach) { + + $content = ''; + + if ($breach->find('h3 > i', 0)) { + + foreach ($breach->find('h3 > i') as $i) { + $content .= $i->title . '.
'; + } + + } + + return $content; + + } + + /** + * Order Breaches by date added or date breached + */ + private function orderBreaches() { + + $sortBy = $this->getInput('order'); + $sort = array(); + + foreach ($this->breaches as $key => $item) { + $sort[$key] = $item[$sortBy]; + } + + array_multisort($sort, SORT_DESC, $this->breaches); + + } + + /** + * Create items from breaches array + */ + private function createItems() { + + $limit = $this->getInput('item_limit'); + + if ($limit < 1) { + $limit = 20; + } + + foreach ($this->breaches as $breach) { + $item = array(); + + $item['title'] = $breach['title']; + $item['timestamp'] = $breach[$this->getInput('order')]; + $item['uri'] = $breach['uri']; + $item['content'] = $breach['content']; + + $this->items[] = $item; + + if (count($this->items) >= $limit) { + break; + } + } + } +} diff --git a/bridges/HeiseBridge.php b/bridges/HeiseBridge.php new file mode 100644 index 0000000..1d9d802 --- /dev/null +++ b/bridges/HeiseBridge.php @@ -0,0 +1,75 @@ + array( + 'name' => 'Category', + 'type' => 'list', + 'values' => array( + 'Alle News' + => 'https://www.heise.de/newsticker/heise-atom.xml', + 'Top-News' + => 'https://www.heise.de/newsticker/heise-top-atom.xml', + 'Internet-Störungen' + => 'https://www.heise.de/netze/netzwerk-tools/imonitor-internet-stoerungen/feed/aktuelle-meldungen/', + 'Alle News von heise Developer' + => 'https://www.heise.de/developer/rss/news-atom.xml' + ) + ), + 'limit' => array( + 'name' => 'Limit', + 'type' => 'number', + 'required' => false, + 'title' => 'Specify number of full articles to return', + 'defaultValue' => 5 + ) + )); + const LIMIT = 5; + + public function collectData() { + $this->collectExpandableDatas( + $this->getInput('category'), + $this->getInput('limit') ?: static::LIMIT + ); + } + + protected function parseItem($feedItem) { + $item = parent::parseItem($feedItem); + $uri = $item['uri']; + + do { + $article = getSimpleHTMLDOMCached($uri) + or returnServerError('Could not open article: ' . $uri); + + $article = defaultLinkTo($article, $uri); + $item = $this->addArticleToItem($item, $article); + + if($next = $article->find('.pagination a[rel="next"]', 0)) + $uri = $next->href; + } while ($next); + + return $item; + } + + private function addArticleToItem($item, $article) { + if($author = $article->find('[itemprop="author"]', 0)) + $item['author'] = $author->plaintext; + + $content = $article->find('div[class*="article-content"]', 0); + + foreach($content->find('p, h3, ul, table, pre, img') as $element) { + $item['content'] .= $element; + } + + foreach($content->find('img') as $img) { + $item['enclosures'][] = $img->src; + } + + return $item; + } +} diff --git a/bridges/HentaiHavenBridge.php b/bridges/HentaiHavenBridge.php index 21a0ff5..0e4fda4 100644 --- a/bridges/HentaiHavenBridge.php +++ b/bridges/HentaiHavenBridge.php @@ -3,7 +3,7 @@ class HentaiHavenBridge extends BridgeAbstract { const MAINTAINER = 'albirew'; const NAME = 'Hentai Haven'; - const URI = 'http://hentaihaven.org/'; + const URI = 'https://hentaihaven.org/'; const CACHE_TIMEOUT = 21600; // 6h const DESCRIPTION = 'Returns releases from Hentai Haven'; diff --git a/bridges/HotUKDealsBridge.php b/bridges/HotUKDealsBridge.php index 8c1b3bd..ed2d28a 100644 --- a/bridges/HotUKDealsBridge.php +++ b/bridges/HotUKDealsBridge.php @@ -17,13 +17,11 @@ class HotUKDealsBridge extends PepperBridgeAbstract { 'hide_expired' => array( 'name' => 'Hide expired deals', 'type' => 'checkbox', - 'required' => true ), 'hide_local' => array( 'name' => 'Hide local deals', 'type' => 'checkbox', 'title' => 'Hide deals in physical store', - 'required' => true ), 'priceFrom' => array( 'name' => 'Minimal Price', @@ -43,7 +41,6 @@ class HotUKDealsBridge extends PepperBridgeAbstract { 'group' => array( 'name' => 'Group', 'type' => 'list', - 'required' => true, 'title' => 'Group whose deals must be displayed', 'values' => array( '2DS' => '2ds', @@ -1317,7 +1314,6 @@ class HotUKDealsBridge extends PepperBridgeAbstract { 'order' => array( 'name' => 'Order by', 'type' => 'list', - 'required' => true, 'title' => 'Sort order of deals', 'values' => array( 'From the most to the least hot deal' => '-hot', diff --git a/bridges/IGNBridge.php b/bridges/IGNBridge.php new file mode 100644 index 0000000..6a254b3 --- /dev/null +++ b/bridges/IGNBridge.php @@ -0,0 +1,55 @@ +collectExpandableDatas('http://feeds.ign.com/ign/all', 15); + } + + // IGNs feed is both hidden and incomplete. This bridge tries to fix this. + + protected function parseItem($newsItem){ + $item = parent::parseItem($newsItem); + + // $articlePage gets the entire page's contents + $articlePage = getSimpleHTMLDOM($newsItem->link); + + /* + * NOTE: Though articles and wiki/howtos have seperate styles of pages, there is no mechanism + * for handling them seperately as it just ignores the DOM querys which it does not find. + * (and their scraping) + */ + + // For Articles + $article = $articlePage->find('section.article-page', 0); + // add in verdicts in articles, reviews etc + foreach($articlePage->find('div.article-section') as $element) { + $article = $article . $element; + } + + // For Wikis and HowTos + $uselessWikiElements = array( + '.wiki-page-tools', + '.feedback-container', + '.paging-container' + ); + foreach($articlePage->find('.wiki-page') as $wikiContents) { + $copy = clone $wikiContents; + // Remove useless elements present in IGN wiki/howtos + foreach($uselessWikiElements as $uslElement) { + $toRemove = $wikiContents->find($uslElement, 0); + $copy = str_replace($toRemove, '', $copy); + } + $article = $article . $copy; + } + + // Add content to feed + $item['content'] = $article; + return $item; + } +} diff --git a/bridges/IndeedBridge.php b/bridges/IndeedBridge.php new file mode 100644 index 0000000..c1d0cfd --- /dev/null +++ b/bridges/IndeedBridge.php @@ -0,0 +1,245 @@ + array( + 'name' => 'Company', + 'type' => 'text', + 'required' => true, + 'title' => 'Company name', + 'exampleValue' => 'GitHub', + ) + ), + 'global' => array( + 'language' => array( + 'name' => 'Language Code', + 'type' => 'list', + 'title' => 'Choose your language code', + 'defaultValue' => 'en-US', + 'values' => array( + 'es-AR' => 'es-AR', + 'de-AT' => 'de-AT', + 'en-AU' => 'en-AU', + 'nl-BE' => 'nl-BE', + 'fr-BE' => 'fr-BE', + 'pt-BR' => 'pt-BR', + 'en-CA' => 'en-CA', + 'fr-CA' => 'fr-CA', + 'de-CH' => 'de-CH', + 'fr-CH' => 'fr-CH', + 'es-CL' => 'es-CL', + 'zh-CN' => 'zh-CN', + 'es-CO' => 'es-CO', + 'de-DE' => 'de-DE', + 'es-ES' => 'es-ES', + 'fr-FR' => 'fr-FR', + 'en-GB' => 'en-GB', + 'en-HK' => 'en-HK', + 'en-IE' => 'en-IE', + 'en-IN' => 'en-IN', + 'it-IT' => 'it-IT', + 'ja-JP' => 'ja-JP', + 'ko-KR' => 'ko-KR', + 'es-MX' => 'es-MX', + 'nl-NL' => 'nl-NL', + 'pl-PL' => 'pl-PL', + 'en-SG' => 'en-SG', + 'en-US' => 'en-US', + 'en-ZA' => 'en-ZA', + 'en-AE' => 'en-AE', + 'da-DK' => 'da-DK', + 'in-ID' => 'in-ID', + 'en-MY' => 'en-MY', + 'es-PE' => 'es-PE', + 'en-PH' => 'en-PH', + 'en-PK' => 'en-PK', + 'ro-RO' => 'ro-RO', + 'ru-RU' => 'ru-RU', + 'tr-TR' => 'tr-TR', + 'zh-TW' => 'zh-TW', + 'vi-VN' => 'vi-VN', + 'en-VN' => 'en-VN', + 'ar-EG' => 'ar-EG', + 'fr-MA' => 'fr-MA', + 'en-NG' => 'en-NG', + ) + ), + 'limit' => array( + 'name' => 'Limit', + 'type' => 'number', + 'title' => 'Maximum number of items to return', + 'exampleValue' => 20, + ) + ) + ); + + const SITES = array( + 'es-AR' => 'https://ar.indeed.com/', + 'de-AT' => 'https://at.indeed.com/', + 'en-AU' => 'https://au.indeed.com/', + 'nl-BE' => 'https://be.indeed.com/', + 'fr-BE' => 'https://emplois.be.indeed.com/', + 'pt-BR' => 'https://www.indeed.com.br/', + 'en-CA' => 'https://ca.indeed.com/', + 'fr-CA' => 'https://emplois.ca.indeed.com/', + 'de-CH' => 'https://www.indeed.ch/', + 'fr-CH' => 'https://emplois.indeed.ch/', + 'es-CL' => 'https://www.indeed.cl/', + 'zh-CN' => 'https://cn.indeed.com/', + 'es-CO' => 'https://co.indeed.com/', + 'de-DE' => 'https://de.indeed.com/', + 'es-ES' => 'https://www.indeed.es/', + 'fr-FR' => 'https://www.indeed.fr/', + 'en-GB' => 'https://www.indeed.co.uk/', + 'en-HK' => 'https://www.indeed.hk/', + 'en-IE' => 'https://ie.indeed.com/', + 'en-IN' => 'https://www.indeed.co.in/', + 'it-IT' => 'https://it.indeed.com/', + 'ja-JP' => 'https://jp.indeed.com/', + 'ko-KR' => 'https://kr.indeed.com/', + 'es-MX' => 'https://www.indeed.com.mx/', + 'nl-NL' => 'https://www.indeed.nl/', + 'pl-PL' => 'https://pl.indeed.com/', + 'en-SG' => 'https://www.indeed.com.sg/', + 'en-US' => 'https://www.indeed.com/', + 'en-ZA' => 'https://www.indeed.co.za/', + 'en-AE' => 'https://www.indeed.ae/', + 'da-DK' => 'https://dk.indeed.com/', + 'in-ID' => 'https://id.indeed.com/', + 'en-MY' => 'https://www.indeed.com.my/', + 'es-PE' => 'https://www.indeed.com.pe/', + 'en-PH' => 'https://www.indeed.com.ph/', + 'en-PK' => 'https://www.indeed.com.pk/', + 'ro-RO' => 'https://ro.indeed.com/', + 'ru-RU' => 'https://ru.indeed.com/', + 'tr-TR' => 'https://tr.indeed.com/', + 'zh-TW' => 'https://tw.indeed.com/', + 'vi-VN' => 'https://vn.indeed.com/', + 'en-VN' => 'https://jobs.vn.indeed.com/', + 'ar-EG' => 'https://eg.indeed.com/', + 'fr-MA' => 'https://ma.indeed.com/', + 'en-NG' => 'https://ng.indeed.com/', + ); + + private $title; + + public function collectData() { + + $url = $this->getURI(); + $limit = $this->getInput('limit') ?: 20; + + do { + + $html = getSimpleHTMLDOM($url) + or returnServerError('Could not request ' . $url); + + $html = defaultLinkTo($html, $url); + + $this->title = $html->find('h1', 0)->innertext; + + // Use local translation of the word "Rating" + $rating_local = $html->find('a[data-id="rating_desc"]', 0)->plaintext; + + foreach($html->find('#cmp-content [id^="cmp-review-"]') as $review) { + $item = array(); + + $rating = $review->find('.cmp-ratingNumber', 0)->plaintext; + $title = $review->find('.cmp-review-title > span', 0)->plaintext; + $comment = $this->beautifyComment($review->find('.cmp-review-content-container', 0)); + + $item['uri'] = $review->find('.cmp-review-share-popup-item-link--copylink', 0)->href; + $item['title'] = "{$rating_local} {$rating} / {$title}"; + $item['timestamp'] = $review->find('.cmp-review-date-created', 0)->plaintext; + $item['author'] = $review->find('.cmp-reviewer', 0)->plaintext; + $item['content'] = $comment; + //$item['enclosures'] + $item['categories'][] = $review->find('.cmp-reviewer-job-location', 0)->plaintext; + //$item['uid'] + + $this->items[] = $item; + + if(count($this->items) >= $limit) { + break; + } + } + + // Break if no more pages available. + if($next = $html->find('a[data-tn-element="next-page"]', 0)) { + $url = $next->href; + } else { + break; + } + + } while(count($this->items) < $limit); + + } + + public function getURI() { + if($this->getInput('language') + && $this->getInput('c')) { + return self::SITES[$this->getInput('language')] + . 'cmp/' + . urlencode($this->getInput('c')) + . '/reviews'; + } + + return parent::getURI(); + } + + public function getName() { + return $this->title ?: parent::getName(); + } + + public function detectParameters($url) { + /** + * Expected: https://<...>.indeed.<...>/cmp/[/reviews][/...] + * + * Note that most users will be redirected to their localized version + * of the page, which adds the language code to the host. For example, + * "en.indeed.com" or "www.indeed.fr" (see link[rel="alternate"]). At + * least each of the sites have ".indeed." in the name. + */ + + if(filter_var($url, FILTER_VALIDATE_URL, FILTER_FLAG_PATH_REQUIRED) === false + || stristr($url, '.indeed.') === false) { + return null; + } + + $url_components = parse_url($url); + $path_segments = array_values(array_filter(explode('/', $url_components['path']))); + + if(count($path_segments) < 2 || $path_segments[0] !== 'cmp') { + return null; + } + + $language = array_search('https://' . $url_components['host'] . '/', self::SITES); + if($language === false) { + return null; + } + + $limit = self::PARAMETERS['global']['limit']['defaultValue'] ?: 20; + $company = $path_segments[1]; + + return array( + 'c' => $company, + 'language' => $language, + 'limit' => $limit, + ); + } + + private function beautifyComment($comment) { + foreach($comment->find('.cmp-bold') as $bold) { + $bold->tag = 'strong'; + $bold->removeClass('cmp-bold'); + } + + return $comment; + } +} diff --git a/bridges/InstagramBridge.php b/bridges/InstagramBridge.php index 317fb12..77a48e6 100644 --- a/bridges/InstagramBridge.php +++ b/bridges/InstagramBridge.php @@ -42,6 +42,38 @@ class InstagramBridge extends BridgeAbstract { ); + const USER_QUERY_HASH = '58b6785bea111c67129decbe6a448951'; + const TAG_QUERY_HASH = '174a5243287c5f3a7de741089750ab3b'; + const STORY_QUERY_HASH = '865589822932d1b43dfe312121dd353a'; + + protected function getInstagramUserId($username) { + + if(is_numeric($username)) return $username; + + $cacheFac = new CacheFactory(); + $cacheFac->setWorkingDir(PATH_LIB_CACHES); + $cache = $cacheFac->create(Configuration::getConfig('cache', 'type')); + $cache->setScope(get_called_class()); + $cache->setKey([$username]); + $key = $cache->loadData(); + + if($key == null) { + $data = getContents(self::URI . 'web/search/topsearch/?query=' . $username); + + foreach(json_decode($data)->users as $user) { + if($user->user->username === $username) { + $key = $user->user->pk; + } + } + if($key == null) { + returnServerError('Unable to find username in search result.'); + } + $cache->saveData($key); + } + return $key; + + } + public function collectData(){ if(is_null($this->getInput('u')) && $this->getInput('media_type') == 'story') { @@ -51,9 +83,9 @@ class InstagramBridge extends BridgeAbstract { $data = $this->getInstagramJSON($this->getURI()); if(!is_null($this->getInput('u'))) { - $userMedia = $data->entry_data->ProfilePage[0]->graphql->user->edge_owner_to_timeline_media->edges; + $userMedia = $data->data->user->edge_owner_to_timeline_media->edges; } elseif(!is_null($this->getInput('h'))) { - $userMedia = $data->entry_data->TagPage[0]->graphql->hashtag->edge_hashtag_to_media->edges; + $userMedia = $data->data->hashtag->edge_hashtag_to_media->edges; } elseif(!is_null($this->getInput('l'))) { $userMedia = $data->entry_data->LocationsPage[0]->graphql->location->edge_location_to_media->edges; } @@ -89,7 +121,7 @@ class InstagramBridge extends BridgeAbstract { if (isset($media->edge_media_to_caption->edges[0]->node->text)) { $textContent = $media->edge_media_to_caption->edges[0]->node->text; } else { - $textContent = basename($media->display_url); + $textContent = '(no text)'; } $item['title'] = ($media->is_video ? '▶ ' : '') . trim($textContent); @@ -99,14 +131,16 @@ class InstagramBridge extends BridgeAbstract { } if(!is_null($this->getInput('u')) && $media->__typename == 'GraphSidecar') { + $data = $this->getInstagramStory($item['uri']); $item['content'] = $data[0]; $item['enclosures'] = $data[1]; } else { + $mediaURI = self::URI . 'p/' . $media->shortcode . '/media?size=l'; $item['content'] = ''; - $item['content'] .= '' . $item['title'] . ''; + $item['content'] .= '' . $item['title'] . ''; $item['content'] .= '

' . nl2br(htmlentities($textContent)); - $item['enclosures'] = array($media->display_url); + $item['enclosures'] = array($mediaURI); } $item['timestamp'] = $media->taken_at_timestamp; @@ -117,8 +151,15 @@ class InstagramBridge extends BridgeAbstract { protected function getInstagramStory($uri) { - $data = $this->getInstagramJSON($uri); - $mediaInfo = $data->entry_data->PostPage[0]->graphql->shortcode_media; + $shortcode = explode('/', $uri)[4]; + $data = getContents(self::URI . + 'graphql/query/?query_hash=' . + self::STORY_QUERY_HASH . + '&variables={"shortcode"%3A"' . + $shortcode . + '"}'); + + $mediaInfo = json_decode($data)->data->shortcode_media; //Process the first element, that isn't in the node graph if (count($mediaInfo->edge_media_to_caption->edges) > 0) { @@ -144,13 +185,38 @@ class InstagramBridge extends BridgeAbstract { protected function getInstagramJSON($uri) { - $html = getContents($uri) - or returnServerError('Could not request Instagram.'); - $scriptRegex = '/window\._sharedData = (.*);<\/script>/'; + if(!is_null($this->getInput('u'))) { + + $userId = $this->getInstagramUserId($this->getInput('u')); - preg_match($scriptRegex, $html, $matches, PREG_OFFSET_CAPTURE, 0); + $data = getContents(self::URI . + 'graphql/query/?query_hash=' . + self::USER_QUERY_HASH . + '&variables={"id"%3A"' . + $userId . + '"%2C"first"%3A10}'); + return json_decode($data); - return json_decode($matches[1][0]); + } elseif(!is_null($this->getInput('h'))) { + $data = getContents(self::URI . + 'graphql/query/?query_hash=' . + self::TAG_QUERY_HASH . + '&variables={"tag_name"%3A"' . + $this->getInput('h') . + '"%2C"first"%3A10}'); + return json_decode($data); + + } else { + + $html = getContents($uri) + or returnServerError('Could not request Instagram.'); + $scriptRegex = '/window\._sharedData = (.*);<\/script>/'; + + preg_match($scriptRegex, $html, $matches, PREG_OFFSET_CAPTURE, 0); + + return json_decode($matches[1][0]); + + } } diff --git a/bridges/InstructablesBridge.php b/bridges/InstructablesBridge.php index 05f1a91..e28c34b 100644 --- a/bridges/InstructablesBridge.php +++ b/bridges/InstructablesBridge.php @@ -1,8 +1,7 @@ array( 'name' => 'Category', 'type' => 'list', - 'required' => true, 'values' => array( - 'Play' => array( - 'All' => '/play/', - 'KNEX' => '/play/knex/', - 'Offbeat' => '/play/offbeat/', - 'Lego' => '/play/lego/', - 'Airsoft' => '/play/airsoft/', - 'Card Games' => '/play/card-games/', - 'Guitars' => '/play/guitars/', - 'Instruments' => '/play/instruments/', - 'Magic Tricks' => '/play/magic-tricks/', - 'Minecraft' => '/play/minecraft/', - 'Music' => '/play/music/', - 'Nerf' => '/play/nerf/', - 'Nintendo' => '/play/nintendo/', - 'Office Supplies' => '/play/office-supplies/', - 'Paintball' => '/play/paintball/', - 'Paper Airplanes' => '/play/paper-airplanes/', - 'Party Tricks' => '/play/party-tricks/', - 'PlayStation' => '/play/playstation/', - 'Pranks and Humor' => '/play/pranks-and-humor/', - 'Puzzles' => '/play/puzzles/', - 'Siege Engines' => '/play/siege-engines/', - 'Sports' => '/play/sports/', - 'Table Top' => '/play/table-top/', - 'Toys' => '/play/toys/', - 'Video Games' => '/play/video-games/', - 'Wii' => '/play/wii/', - 'Xbox' => '/play/xbox/', - 'Yo-Yo' => '/play/yo-yo/', + 'Circuits' => array( + 'All' => '/circuits/', + 'Apple' => '/circuits/apple/projects/', + 'Arduino' => '/circuits/arduino/projects/', + 'Art' => '/circuits/art/projects/', + 'Assistive Tech' => '/circuits/assistive-tech/projects/', + 'Audio' => '/circuits/audio/projects/', + 'Cameras' => '/circuits/cameras/projects/', + 'Clocks' => '/circuits/clocks/projects/', + 'Computers' => '/circuits/computers/projects/', + 'Electronics' => '/circuits/electronics/projects/', + 'Gadgets' => '/circuits/gadgets/projects/', + 'Lasers' => '/circuits/lasers/projects/', + 'LEDs' => '/circuits/leds/projects/', + 'Linux' => '/circuits/linux/projects/', + 'Microcontrollers' => '/circuits/microcontrollers/projects/', + 'Microsoft' => '/circuits/microsoft/projects/', + 'Mobile' => '/circuits/mobile/projects/', + 'Raspberry Pi' => '/circuits/raspberry-pi/projects/', + 'Remote Control' => '/circuits/remote-control/projects/', + 'Reuse' => '/circuits/reuse/projects/', + 'Robots' => '/circuits/robots/projects/', + 'Sensors' => '/circuits/sensors/projects/', + 'Software' => '/circuits/software/projects/', + 'Soldering' => '/circuits/soldering/projects/', + 'Speakers' => '/circuits/speakers/projects/', + 'Tools' => '/circuits/tools/projects/', + 'USB' => '/circuits/usb/projects/', + 'Wearables' => '/circuits/wearables/projects/', + 'Websites' => '/circuits/websites/projects/', + 'Wireless' => '/circuits/wireless/projects/', + ), + 'Workshop' => array( + 'All' => '/workshop/', + '3D Printing' => '/workshop/3d-printing/projects/', + 'Cars' => '/workshop/cars/projects/', + 'CNC' => '/workshop/cnc/projects/', + 'Electric Vehicles' => '/workshop/electric-vehicles/projects/', + 'Energy' => '/workshop/energy/projects/', + 'Furniture' => '/workshop/furniture/projects/', + 'Home Improvement' => '/workshop/home-improvement/projects/', + 'Home Theater' => '/workshop/home-theater/projects/', + 'Hydroponics' => '/workshop/hydroponics/projects/', + 'Knives' => '/workshop/knives/projects/', + 'Laser Cutting' => '/workshop/laser-cutting/projects/', + 'Lighting' => '/workshop/lighting/projects/', + 'Metalworking' => '/workshop/metalworking/projects/', + 'Molds & Casting' => '/workshop/molds-and-casting/projects/', + 'Motorcycles' => '/workshop/motorcycles/projects/', + 'Organizing' => '/workshop/organizing/projects/', + 'Pallets' => '/workshop/pallets/projects/', + 'Repair' => '/workshop/repair/projects/', + 'Science' => '/workshop/science/projects/', + 'Shelves' => '/workshop/shelves/projects/', + 'Solar' => '/workshop/solar/projects/', + 'Tools' => '/workshop/tools/projects/', + 'Woodworking' => '/workshop/woodworking/projects/', + 'Workbenches' => '/workshop/workbenches/projects/', ), 'Craft' => array( 'All' => '/craft/', - 'Art' => '/craft/art/', - 'Sewing' => '/craft/sewing/', - 'Paper' => '/craft/paper/', - 'Jewelry' => '/craft/jewelry/', - 'Fashion' => '/craft/fashion/', - 'Books & Journals' => '/craft/books-and-journals/', - 'Cards' => '/craft/cards/', - 'Clay' => '/craft/clay/', - 'Duct Tape' => '/craft/duct-tape/', - 'Embroidery' => '/craft/embroidery/', - 'Felt' => '/craft/felt/', - 'Fiber Arts' => '/craft/fiber-arts/', - 'Gifts & Wrapping' => '/craft/gifts-and-wrapping/', - 'Knitting & Crocheting' => '/craft/knitting-and-crocheting/', - 'Leather' => '/craft/leather/', - 'Mason Jars' => '/craft/mason-jars/', - 'No-Sew' => '/craft/no-sew/', - 'Parties & Weddings' => '/craft/parties-and-weddings/', - 'Print Making' => '/craft/print-making/', - 'Soap' => '/craft/soap/', - 'Wallets' => '/craft/wallets/', + 'Art' => '/craft/art/projects/', + 'Books & Journals' => '/craft/books-and-journals/projects/', + 'Cardboard' => '/craft/cardboard/projects/', + 'Cards' => '/craft/cards/projects/', + 'Clay' => '/craft/clay/projects/', + 'Costumes & Cosplay' => '/craft/costumes-and-cosplay/projects/', + 'Digital Graphics' => '/craft/digital-graphics/projects/', + 'Duct Tape' => '/craft/duct-tape/projects/', + 'Embroidery' => '/craft/embroidery/projects/', + 'Fashion' => '/craft/fashion/projects/', + 'Felt' => '/craft/felt/projects/', + 'Fiber Arts' => '/craft/fiber-arts/projects/', + 'Gift Wrapping' => '/craft/gift-wrapping/projects/', + 'Jewelry' => '/craft/jewelry/projects/', + 'Knitting & Crochet' => '/craft/knitting-and-crochet/projects/', + 'Leather' => '/craft/leather/projects/', + 'Mason Jars' => '/craft/mason-jars/projects/', + 'No-Sew' => '/craft/no-sew/projects/', + 'Paper' => '/craft/paper/projects/', + 'Parties & Weddings' => '/craft/parties-and-weddings/projects/', + 'Photography' => '/craft/photography/projects/', + 'Printmaking' => '/craft/printmaking/projects/', + 'Reuse' => '/craft/reuse/projects/', + 'Sewing' => '/craft/sewing/projects/', + 'Soapmaking' => '/craft/soapmaking/projects/', + 'Wallets' => '/craft/wallets/projects/', ), - 'Technology' => array( - 'All' => '/technology/', - 'Electronics' => '/technology/electronics/', - 'Arduino' => '/technology/arduino/', - 'Photography' => '/technology/photography/', - 'Leds' => '/technology/leds/', - 'Science' => '/technology/science/', - 'Reuse' => '/technology/reuse/', - 'Apple' => '/technology/apple/', - 'Computers' => '/technology/computers/', - '3D Printing' => '/technology/3D-Printing/', - 'Robots' => '/technology/robots/', - 'Art' => '/technology/art/', - 'Assistive Tech' => '/technology/assistive-technology/', - 'Audio' => '/technology/audio/', - 'Clocks' => '/technology/clocks/', - 'CNC' => '/technology/cnc/', - 'Digital Graphics' => '/technology/digital-graphics/', - 'Gadgets' => '/technology/gadgets/', - 'Kits' => '/technology/kits/', - 'Laptops' => '/technology/laptops/', - 'Lasers' => '/technology/lasers/', - 'Linux' => '/technology/linux/', - 'Microcontrollers' => '/technology/microcontrollers/', - 'Microsoft' => '/technology/microsoft/', - 'Mobile' => '/technology/mobile/', - 'Raspberry Pi' => '/technology/raspberry-pi/', - 'Remote Control' => '/technology/remote-control/', - 'Sensors' => '/technology/sensors/', - 'Software' => '/technology/software/', - 'Soldering' => '/technology/soldering/', - 'Speakers' => '/technology/speakers/', - 'Steampunk' => '/technology/steampunk/', - 'Tools' => '/technology/tools/', - 'USB' => '/technology/usb/', - 'Wearables' => '/technology/wearables/', - 'Websites' => '/technology/websites/', - 'Wireless' => '/technology/wireless/', - ), - 'Workshop' => array( - 'All' => '/workshop/', - 'Woodworking' => '/workshop/woodworking/', - 'Tools' => '/workshop/tools/', - 'Gardening' => '/workshop/gardening/', - 'Cars' => '/workshop/cars/', - 'Metalworking' => '/workshop/metalworking/', - 'Cardboard' => '/workshop/cardboard/', - 'Electric Vehicles' => '/workshop/electric-vehicles/', - 'Energy' => '/workshop/energy/', - 'Furniture' => '/workshop/furniture/', - 'Home Improvement' => '/workshop/home-improvement/', - 'Home Theater' => '/workshop/home-theater/', - 'Hydroponics' => '/workshop/hydroponics/', - 'Laser Cutting' => '/workshop/laser-cutting/', - 'Lighting' => '/workshop/lighting/', - 'Molds & Casting' => '/workshop/molds-and-casting/', - 'Motorcycles' => '/workshop/motorcycles/', - 'Organizing' => '/workshop/organizing/', - 'Pallets' => '/workshop/pallets/', - 'Repair' => '/workshop/repair/', - 'Shelves' => '/workshop/shelves/', - 'Solar' => '/workshop/solar/', - 'Workbenches' => '/workshop/workbenches/', + 'Cooking' => array( + 'All' => '/cooking/', + 'Bacon' => '/cooking/bacon/projects/', + 'BBQ & Grilling' => '/cooking/bbq-and-grilling/projects/', + 'Beverages' => '/cooking/beverages/projects/', + 'Bread' => '/cooking/bread/projects/', + 'Breakfast' => '/cooking/breakfast/projects/', + 'Cake' => '/cooking/cake/projects/', + 'Candy' => '/cooking/candy/projects/', + 'Canning & Preserving' => '/cooking/canning-and-preserving/projects/', + 'Cocktails & Mocktails' => '/cooking/cocktails-and-mocktails/projects/', + 'Coffee' => '/cooking/coffee/projects/', + 'Cookies' => '/cooking/cookies/projects/', + 'Cupcakes' => '/cooking/cupcakes/projects/', + 'Dessert' => '/cooking/dessert/projects/', + 'Homebrew' => '/cooking/homebrew/projects/', + 'Main Course' => '/cooking/main-course/projects/', + 'Pasta' => '/cooking/pasta/projects/', + 'Pie' => '/cooking/pie/projects/', + 'Pizza' => '/cooking/pizza/projects/', + 'Salad' => '/cooking/salad/projects/', + 'Sandwiches' => '/cooking/sandwiches/projects/', + 'Snacks & Appetizers' => '/cooking/snacks-and-appetizers/projects/', + 'Soups & Stews' => '/cooking/soups-and-stews/projects/', + 'Vegetarian & Vegan' => '/cooking/vegetarian-and-vegan/projects/', ), - 'Home' => array( - 'All' => '/home/', - 'Halloween' => '/home/halloween/', - 'Decorating' => '/home/decorating/', - 'Organizing' => '/home/organizing/', - 'Pets' => '/home/pets/', - 'Life Hacks' => '/home/life-hacks/', - 'Beauty' => '/home/beauty/', - 'Christmas' => '/home/christmas/', - 'Cleaning' => '/home/cleaning/', - 'Education' => '/home/education/', - 'Finances' => '/home/finances/', - 'Gardening' => '/home/gardening/', - 'Green' => '/home/green/', - 'Health' => '/home/health/', - 'Hiding Places' => '/home/hiding-places/', - 'Holidays' => '/home/holidays/', - 'Homesteading' => '/home/homesteading/', - 'Kids' => '/home/kids/', - 'Kitchen' => '/home/kitchen/', - 'Life Skills' => '/home/life-skills/', - 'Parenting' => '/home/parenting/', - 'Pest Control' => '/home/pest-control/', - 'Relationships' => '/home/relationships/', - 'Reuse' => '/home/reuse/', - 'Travel' => '/home/travel/', + 'Living' => array( + 'All' => '/living/', + 'Beauty' => '/living/beauty/projects/', + 'Christmas' => '/living/christmas/projects/', + 'Cleaning' => '/living/cleaning/projects/', + 'Decorating' => '/living/decorating/projects/', + 'Education' => '/living/education/projects/', + 'Gardening' => '/living/gardening/projects/', + 'Halloween' => '/living/halloween/projects/', + 'Health' => '/living/health/projects/', + 'Hiding Places' => '/living/hiding-places/projects/', + 'Holidays' => '/living/holidays/projects/', + 'Homesteading' => '/living/homesteading/projects/', + 'Kids' => '/living/kids/projects/', + 'Kitchen' => '/living/kitchen/projects/', + 'LEGO & KNEX' => '/living/lego-and-knex/projects/', + 'Life Hacks' => '/living/life-hacks/projects/', + 'Music' => '/living/music/projects/', + 'Office Supply Hacks' => '/living/office-supply-hacks/projects/', + 'Organizing' => '/living/organizing/projects/', + 'Pest Control' => '/living/pest-control/projects/', + 'Pets' => '/living/pets/projects/', + 'Pranks, Tricks, & Humor' => '/living/pranks-tricks-and-humor/projects/', + 'Relationships' => '/living/relationships/projects/', + 'Toys & Games' => '/living/toys-and-games/projects/', + 'Travel' => '/living/travel/projects/', + 'Video Games' => '/living/video-games/projects/', ), 'Outside' => array( 'All' => '/outside/', - 'Bikes' => '/outside/bikes/', - 'Survival' => '/outside/survival/', - 'Backyard' => '/outside/backyard/', - 'Beach' => '/outside/beach/', - 'Birding' => '/outside/birding/', - 'Boats' => '/outside/boats/', - 'Camping' => '/outside/camping/', - 'Climbing' => '/outside/climbing/', - 'Fire' => '/outside/fire/', - 'Fishing' => '/outside/fishing/', - 'Hunting' => '/outside/hunting/', - 'Kites' => '/outside/kites/', - 'Knives' => '/outside/knives/', - 'Knots' => '/outside/knots/', - 'Paracord' => '/outside/paracord/', - 'Rockets' => '/outside/rockets/', - 'Skateboarding' => '/outside/skateboarding/', - 'Snow' => '/outside/snow/', - 'Water' => '/outside/water/', + 'Backyard' => '/outside/backyard/projects/', + 'Beach' => '/outside/beach/projects/', + 'Bikes' => '/outside/bikes/projects/', + 'Birding' => '/outside/birding/projects/', + 'Boats' => '/outside/boats/projects/', + 'Camping' => '/outside/camping/projects/', + 'Climbing' => '/outside/climbing/projects/', + 'Fire' => '/outside/fire/projects/', + 'Fishing' => '/outside/fishing/projects/', + 'Hunting' => '/outside/hunting/projects/', + 'Kites' => '/outside/kites/projects/', + 'Knots' => '/outside/knots/projects/', + 'Launchers' => '/outside/launchers/projects/', + 'Paracord' => '/outside/paracord/projects/', + 'Rockets' => '/outside/rockets/projects/', + 'Siege Engines' => '/outside/siege-engines/projects/', + 'Skateboarding' => '/outside/skateboarding/projects/', + 'Snow' => '/outside/snow/projects/', + 'Sports' => '/outside/sports/projects/', + 'Survival' => '/outside/survival/projects/', + 'Water' => '/outside/water/projects/', + ), + 'Makeymakey' => array( + 'All' => '/makeymakey/', + 'Makey Makey on Instructables' => '/makeymakey/', ), - 'Food' => array( - 'All' => '/food/', - 'Dessert' => '/food/dessert/', - 'Snacks & Appetizers' => '/food/snacks-and-appetizers/', - 'Bacon' => '/food/bacon/', - 'BBQ & Grilling' => '/food/bbq-and-grilling/', - 'Beverages' => '/food/beverages/', - 'Bread' => '/food/bread/', - 'Breakfast' => '/food/breakfast/', - 'Cake' => '/food/cake/', - 'Candy' => '/food/candy/', - 'Canning & Preserves' => '/food/canning-and-preserves/', - 'Cocktails & Mocktails' => '/food/cocktails-and-mocktails/', - 'Coffee' => '/food/coffee/', - 'Cookies' => '/food/cookies/', - 'Cupcakes' => '/food/cupcakes/', - 'Homebrew' => '/food/homebrew/', - 'Main Course' => '/food/main-course/', - 'Pasta' => '/food/pasta/', - 'Pie' => '/food/pie/', - 'Pizza' => '/food/pizza/', - 'Salad' => '/food/salad/', - 'Sandwiches' => '/food/sandwiches/', - 'Soups & Stews' => '/food/soups-and-stews/', - 'Vegetarian & Vegan' => '/food/vegetarian-and-vegan/', + 'Teachers' => array( + 'All' => '/teachers/', + 'ELA' => '/teachers/ela/projects/', + 'Math' => '/teachers/math/projects/', + 'Science' => '/teachers/science/projects/', + 'Social Studies' => '/teachers/social-studies/projects/', + 'Engineering' => '/teachers/engineering/projects/', + 'Coding' => '/teachers/coding/projects/', + 'Electronics' => '/teachers/electronics/projects/', + 'Robotics' => '/teachers/robotics/projects/', + 'Arduino' => '/teachers/arduino/projects/', + 'CNC' => '/teachers/cnc/projects/', + 'Laser Cutting' => '/teachers/laser-cutting/projects/', + '3D Printing' => '/teachers/3d-printing/projects/', + '3D Design' => '/teachers/3d-design/projects/', + 'Art' => '/teachers/art/projects/', + 'Music' => '/teachers/music/projects/', + 'Theatre' => '/teachers/theatre/projects/', + 'Wood Shop' => '/teachers/wood-shop/projects/', + 'Metal Shop' => '/teachers/metal-shop/projects/', + 'Resources' => '/teachers/resources/projects/', ), - 'Costumes' => array( - 'All' => '/costumes/', - 'Props' => '/costumes/props-and-accessories/', - 'Animals' => '/costumes/animals/', - 'Comics' => '/costumes/comics/', - 'Fantasy' => '/costumes/fantasy/', - 'For Kids' => '/costumes/for-kids/', - 'For Pets' => '/costumes/for-pets/', - 'Funny' => '/costumes/funny/', - 'Games' => '/costumes/games/', - 'Historic & Futuristic' => '/costumes/historic-and-futuristic/', - 'Makeup' => '/costumes/makeup/', - 'Masks' => '/costumes/masks/', - 'Scary' => '/costumes/scary/', - 'TV & Movies' => '/costumes/tv-and-movies/', - 'Weapons & Armor' => '/costumes/weapons-and-armor/', - ) ), 'title' => 'Select your category (required)', - 'defaultValue' => 'Technology' + 'defaultValue' => 'Circuits' ), 'filter' => array( 'name' => 'Filter', 'type' => 'list', - 'required' => true, 'values' => array( 'Featured' => ' ', 'Recent' => 'recent/', @@ -254,65 +233,70 @@ class InstructablesBridge extends BridgeAbstract { ) ); - private $uri; - public function collectData() { // Enable the following line to get the category list (dev mode) // $this->listCategories(); - $this->uri = static::URI; - - switch($this->queriedContext) { - case 'Category': $this->uri .= $this->getInput('category') . $this->getInput('filter'); - } + $html = getSimpleHTMLDOM($this->getURI()) + or returnServerError('Error loading category ' . $this->getURI()); + $html = defaultLinkTo($html, $this->getURI()); - $html = getSimpleHTMLDOM($this->uri) - or returnServerError('Error loading category ' . $this->uri); + $covers = $html->find(' + .category-projects-list > div, + .category-landing-projects-list > div, + '); - foreach($html->find('ul.explore-covers-list li') as $cover) { + foreach($covers as $cover) { $item = array(); - $item['uri'] = static::URI . $cover->find('a.cover-image', 0)->href; - $item['title'] = $cover->find('.title', 0)->innertext; + $item['uri'] = $cover->find('a.ible-title', 0)->href; + $item['title'] = $cover->find('a.ible-title', 0)->innertext; $item['author'] = $this->getCategoryAuthor($cover); $item['content'] = 'src + . $cover->find('img', 0)->getAttribute('data-src') . '>'; - $image = str_replace('.RECTANGLE1', '.LARGE', $cover->find('a.cover-image img', 0)->src); - $item['enclosures'] = [$image]; + $item['enclosures'][] = str_replace( + '.RECTANGLE1', + '.LARGE', + $cover->find('img', 0)->getAttribute('data-src') + ); $this->items[] = $item; } } public function getName() { - if(!is_null($this->getInput('category')) - && !is_null($this->getInput('filter'))) { - foreach(self::PARAMETERS[$this->queriedContext]['category']['values'] as $key => $value) { - $subcategory = array_search($this->getInput('category'), $value); + switch($this->queriedContext) { + case 'Category': { + foreach(self::PARAMETERS[$this->queriedContext]['category']['values'] as $key => $value) { + $subcategory = array_search($this->getInput('category'), $value); - if($subcategory !== false) - break; - } + if($subcategory !== false) + break; + } - $filter = array_search( - $this->getInput('filter'), - self::PARAMETERS[$this->queriedContext]['filter']['values'] - ); + $filter = array_search( + $this->getInput('filter'), + self::PARAMETERS[$this->queriedContext]['filter']['values'] + ); - return $subcategory . ' (' . $filter . ') - ' . static::NAME; + return $subcategory . ' (' . $filter . ') - ' . static::NAME; + } break; } return parent::getName(); } public function getURI() { - if(!is_null($this->getInput('category')) - && !is_null($this->getInput('filter'))) { - return $this->uri; + switch($this->queriedContext) { + case 'Category': { + return self::URI + . $this->getInput('category') + . $this->getInput('filter'); + } break; } return parent::getURI(); @@ -323,24 +307,32 @@ class InstructablesBridge extends BridgeAbstract { * parameters list) */ private function listCategories(){ - // Use arbitrary category to receive full list - $html = getSimpleHTMLDOM(self::URI . '/technology/'); - foreach($html->find('.channel a') as $channel) { - $name = html_entity_decode(trim($channel->innertext)); + // Use home page to acquire main categories + $html = getSimpleHTMLDOM(self::URI); + $html = defaultLinkTo($html, self::URI); + + foreach($html->find('.home-content-explore-link') as $category) { + + // Use arbitrary category to receive full list + $html = getSimpleHTMLDOM($category->href); - // Remove unwanted entities - $name = str_replace("'", '', $name); - $name = str_replace(''', '', $name); + foreach($html->find('.channel-thumbnail a') as $channel) { + $name = html_entity_decode(trim($channel->title)); - $uri = $channel->href; + // Remove unwanted entities + $name = str_replace("'", '', $name); + $name = str_replace(''', '', $name); - $category = explode('/', $uri)[1]; + $uri = $channel->href; - if(!isset($categories) - || !array_key_exists($category, $categories) - || !in_array($uri, $categories[$category])) - $categories[$category][$name] = $uri; + $category_name = explode('/', $uri)[1]; + + if(!isset($categories) + || !array_key_exists($category_name, $categories) + || !in_array($uri, $categories[$category_name])) + $categories[$category_name][$name] = $uri; + } } // Build PHP array manually @@ -362,9 +354,9 @@ class InstructablesBridge extends BridgeAbstract { */ private function getCategoryAuthor($cover) { return 'href + . $cover->find('.ible-author a', 0)->href . '>' - . $cover->find('span.author a', 0)->innertext + . $cover->find('.ible-author a', 0)->innertext . ''; } } diff --git a/bridges/InternetArchiveBridge.php b/bridges/InternetArchiveBridge.php new file mode 100644 index 0000000..dca1c32 --- /dev/null +++ b/bridges/InternetArchiveBridge.php @@ -0,0 +1,293 @@ + array( + 'username' => array( + 'name' => 'Username', + 'type' => 'text', + 'required' => true, + 'exampleValue' => '@verifiedjoseph', + ), + 'content' => array( + 'name' => 'Content', + 'type' => 'list', + 'values' => array( + 'Uploads' => 'uploads', + 'Posts' => 'posts', + 'Reviews' => 'reviews', + 'Collections' => 'collections', + 'Web Archives' => 'web-archive', + ), + 'defaultValue' => 'uploads', + ) + ) + ); + + const CACHE_TIMEOUT = 900; // 15 mins + + private $skipClasses = array( + 'item-ia mobile-header hidden-tiles', + 'item-ia account-ia' + ); + + public function collectData() { + + $html = getSimpleHTMLDOM($this->getURI()) + or returnServerError('Could not request: ' . $this->getURI()); + + $html = defaultLinkTo($html, $this->getURI()); + + if ($this->getInput('content') !== 'posts') { + + $detailsDivNumber = 0; + + foreach ($html->find('div.results > div[data-id]') as $index => $result) { + $item = array(); + + if (in_array($result->class, $this->skipClasses)) { + continue; + } + + switch($result->class) { + case 'item-ia': + + switch($this->getInput('content')) { + case 'reviews': + $item = $this->processReview($result); + break; + case 'uploads': + $item = $this->processUpload($result); + break; + } + + break; + case 'item-ia url-item': + $item = $this->processWebArchives($result); + break; + case 'item-ia collection-ia': + $item = $this->processCollection($result); + break; + } + + if ($this->getInput('content') !== 'reviews') { + $hiddenDetails = $this->processHiddenDetails($html, $detailsDivNumber, $item); + + $this->items[] = array_merge($item, $hiddenDetails); + } else { + + $this->items[] = $item; + + } + + $detailsDivNumber++; + } + } + + if ($this->getInput('content') === 'posts') { + $this->items = $this->processPosts($html); + } + } + + public function getURI() { + + if (!is_null($this->getInput('username')) && !is_null($this->getInput('content'))) { + return self::URI . '/details/' . $this->processUsername() . '&tab=' . $this->getInput('content'); + } + + return parent::getURI(); + } + + public function getName() { + + if (!is_null($this->getInput('username')) && !is_null($this->getInput('content'))) { + + $contentValues = array_flip(self::PARAMETERS['Account']['content']['values']); + + return $contentValues[$this->getInput('content')] . ' - ' + . $this->processUsername() . ' - Internet Archive'; + } + + return parent::getName(); + } + + private function processUsername() { + + if (substr($this->getInput('username'), 0, 1) !== '@') { + return '@' . $this->getInput('username'); + } + + return $this->getInput('username'); + } + + private function processUpload($result) { + + $item = array(); + + $collection = $result->find('a.stealth', 0); + $collectionLink = self::URI . $collection->href; + $collectionTitle = $collection->find('div.item-parent-ttl', 0)->plaintext; + + $item['title'] = trim($result->find('div.ttl', 0)->innertext); + $item['timestamp'] = strtotime($result->find('div.hidden-tiles.pubdate.C.C3', 0)->children(0)->plaintext); + $item['uri'] = $result->find('div.item-ttl.C.C2 > a', 0)->href; + + if ($result->find('div.by.C.C4', 0)->children(2)) { + $item['author'] = $result->find('div.by.C.C4', 0)->children(2)->plaintext; + } + + $item['content'] = <<Media Type: {$result->attr['data-mediatype']}
+Collection: {$collectionTitle}

+EOD; + + $item['enclosures'][] = self::URI . $result->find('img.item-img', 0)->source; + + return $item; + } + + private function processReview($result) { + + $item = array(); + + $item['title'] = trim($result->find('div.ttl', 0)->innertext); + $item['timestamp'] = strtotime($result->find('div.hidden-tiles.pubdate.C.C3', 0)->children(0)->plaintext); + $item['uri'] = $result->find('div.review-title', 0)->children(0)->href; + + if ($result->find('div.by.C.C4', 0)->children(2)) { + $item['author'] = $result->find('div.by.C.C4', 0)->children(2)->plaintext; + } + + $item['content'] = <<Subject: {$result->find('div.review-title', 0)->plaintext}

+

{$result->find('div.hidden-lists.review' , 0)->children(1)->plaintext}

+EOD; + + $item['enclosures'][] = self::URI . $result->find('img.item-img', 0)->source; + + return $item; + } + + private function processWebArchives($result) { + + $item = array(); + + $item['title'] = trim($result->find('div.ttl', 0)->plaintext); + $item['timestamp'] = strtotime($result->find('div.hidden-lists', 0)->children(0)->plaintext); + $item['uri'] = $result->find('div.item-ttl.C.C2 > a', 0)->href; + + $item['content'] = <<processUsername()} archived {$result->find('div.ttl', 0)->plaintext} +EOD; + + $item['enclosures'][] = $result->find('img.item-img', 0)->source; + + return $item; + } + + private function processCollection($result) { + + $item = array(); + + $title = trim($result->find('div.collection-title.C.C2', 0)->children(0)->plaintext); + $itemCount = strtolower(trim($result->find('div.num-items.topinblock', 0)->plaintext)); + + $item['title'] = $title . ' (' . $itemCount . ')'; + $item['timestamp'] = strtotime($result->find('div.hidden-tiles.pubdate.C.C3', 0)->children(0)->plaintext); + $item['uri'] = $result->find('div.collection-title.C.C2 > a', 0)->href; + + $item['content'] = ''; + + if ($result->find('img.item-img', 0)) { + $item['enclosures'][] = self::URI . $result->find('img.item-img', 0)->source; + } + + return $item; + } + + private function processHiddenDetails($html, $detailsDivNumber, $item) { + + $description = ''; + + if ($html->find('div.details-ia.hidden-tiles', $detailsDivNumber)) { + $detailsDiv = $html->find('div.details-ia.hidden-tiles', $detailsDivNumber); + + if ($detailsDiv->find('div.C234', 0)->children(0)) { + $description = $detailsDiv->find('div.C234', 0)->children(0)->plaintext; + + $detailsDiv->find('div.C234', 0)->children(0)->innertext = ''; + } + + $topics = trim($detailsDiv->find('div.C234', 0)->plaintext); + + if (!empty($topics)) { + $topics = trim($detailsDiv->find('div.C234', 0)->plaintext); + $topics = trim(substr($topics, 7)); + + $item['categories'] = explode(',', $topics); + } + + $item['content'] = '

' . $description . '

' . $item['content']; + } + + return $item; + } + + private function processPosts($html) { + + $items = array(); + + foreach ($html->find('table.forumTable > tr') as $index => $tr) { + $item = array(); + + if ($index === 0) { + continue; + } + + $item['title'] = $tr->find('td', 0)->plaintext; + $item['timestamp'] = strtotime($tr->find('td', 4)->children(0)->plaintext); + $item['uri'] = $tr->find('td', 0)->children(0)->href; + + $formLink = <<{$tr->find('td', 2)->children(0)->plaintext} +EOD; + + $postDate = $tr->find('td', 4)->children(0)->plaintext; + + $postPageHtml = getSimpleHTMLDOMCached($item['uri'], 3600) + or returnServerError('Could not request: ' . $item['uri']); + + $postPageHtml = defaultLinkTo($postPageHtml, $this->getURI()); + + $post = $postPageHtml->find('div.box.well.well-sm', 0); + + $parentLink = ''; + $replyLink = <<Reply +EOD; + + if ($post->find('a', 1)->innertext = 'See parent post') { + $parentLink = <<View parent post +EOD; + } + + $post->find('h1', 0)->outertext = ''; + $post->find('h2', 0)->outertext = ''; + + $item['content'] = <<{$post->innertext}

{$replyLink} - {$parentLink} - Posted in {$formLink} on {$postDate} +EOD; + + $items[] = $item; + + if (count($items) >= 10) { + break; + } + } + return $items; + } +} diff --git a/bridges/IvooxBridge.php b/bridges/IvooxBridge.php new file mode 100644 index 0000000..3cdf74b --- /dev/null +++ b/bridges/IvooxBridge.php @@ -0,0 +1,128 @@ + array( + 's' => array( + 'name' => 'keyword', + 'exampleValue' => 'test' + ) + ) + ); + + private function ivBridgeAddItem( + $episode_link, + $podcast_name, + $episode_title, + $author_name, + $episode_description, + $publication_date, + $episode_duration) { + $item = array(); + $item['title'] = htmlspecialchars_decode($podcast_name . ': ' . $episode_title); + $item['author'] = $author_name; + $item['timestamp'] = $publication_date; + $item['uri'] = $episode_link; + $item['content'] = '' . $podcast_name . ': ' . $episode_title + . '
Duration: ' . $episode_duration + . '
Description:
' . $episode_description; + $this->items[] = $item; + } + + private function ivBridgeParseHtmlListing($html) { + $limit = 4; + $count = 0; + + foreach($html->find('div.flip-container') as $flipper) { + $linkcount = 0; + if(!empty($flipper->find( 'div.modulo-type-banner' ))) { + // ad + continue; + } + + if($count < $limit) { + foreach($flipper->find('div.header-modulo') as $element) { + foreach($element->find('a') as $link) { + if ($linkcount == 0) { + $episode_link = $link->href; + $episode_title = $link->title; + } elseif ($linkcount == 1) { + $author_link = $link->href; + $author_name = $link->title; + } elseif ($linkcount == 2) { + $podcast_link = $link->href; + $podcast_name = $link->title; + } + + $linkcount++; + } + } + + $episode_description = $flipper->find('button.btn-link', 0)->getAttribute('data-content'); + $episode_duration = $flipper->find('p.time', 0)->innertext; + $publication_date = $flipper->find('li.date', 0)->getAttribute('title'); + + // alternative date_parse_from_format + // or DateTime::createFromFormat('G:i - d \d\e M \d\e Y', $publication); + // TODO: month name translations, due function doesn't support locale + + $a = strptime($publication_date, '%H:%M - %d de %b. de %Y'); // obsolete function, uses c libraries + $publication_date = mktime(0, 0, 0, $a['tm_mon'] + 1, $a['tm_mday'], $a['tm_year'] + 1900); + + $this->ivBridgeAddItem( + $episode_link, + $podcast_name, + $episode_title, + $author_name, + $episode_description, + $publication_date, + $episode_duration + ); + $count++; + } + } + } + + public function collectData() { + + // store locale, change to spanish + $originalLocales = explode(';', setlocale(LC_ALL, 0)); + setlocale(LC_ALL, 'es_ES.utf8'); + + $xml = ''; + $html = ''; + $url_feed = ''; + if($this->getInput('s')) { /* Search modes */ + $this->request = str_replace(' ', '-', $this->getInput('s')); + $url_feed = self::URI . urlencode($this->request) . '_sb_f_1.html?o=uploaddate'; + } else { + returnClientError('Not valid mode at IvooxBridge'); + } + + $dom = getSimpleHTMLDOM($url_feed) + or returnServerError('Could not request ' . $url_feed); + $this->ivBridgeParseHtmlListing($dom); + + // restore locale + + foreach($originalLocales as $localeSetting) { + if(strpos($localeSetting, '=') !== false) { + list($category, $locale) = explode('=', $localeSetting); + } else { + $category = LC_ALL; + $locale = $localeSetting; + } + + setlocale($category, $locale); + } + } +} diff --git a/bridges/JustETFBridge.php b/bridges/JustETFBridge.php index 85318b8..8d5b3d5 100644 --- a/bridges/JustETFBridge.php +++ b/bridges/JustETFBridge.php @@ -34,7 +34,6 @@ class JustETFBridge extends BridgeAbstract { 'global' => array( 'lang' => array( 'name' => 'Language', - 'required' => true, 'type' => 'list', 'values' => array( 'Englisch' => 'en', diff --git a/bridges/KununuBridge.php b/bridges/KununuBridge.php index 2f4bf0b..7cc4af6 100644 --- a/bridges/KununuBridge.php +++ b/bridges/KununuBridge.php @@ -11,7 +11,6 @@ class KununuBridge extends BridgeAbstract { 'site' => array( 'name' => 'Site', 'type' => 'list', - 'required' => true, 'title' => 'Select your site', 'values' => array( 'Austria' => 'at', @@ -23,9 +22,18 @@ class KununuBridge extends BridgeAbstract { 'full' => array( 'name' => 'Load full article', 'type' => 'checkbox', - 'required' => false, 'exampleValue' => 'checked', 'title' => 'Activate to load full article' + ), + 'include_ratings' => array( + 'name' => 'Include ratings', + 'type' => 'checkbox', + 'title' => 'Activate to include ratings in the feed' + ), + 'include_benefits' => array( + 'name' => 'Include benefits', + 'type' => 'checkbox', + 'title' => 'Activate to include benefits in the feed' ) ), array( @@ -118,7 +126,7 @@ class KununuBridge extends BridgeAbstract { $item = array(); $item['author'] = $this->extractArticleAuthorPosition($article); - $item['timestamp'] = strtotime($date); + $item['timestamp'] = strtotime($date->content); $item['title'] = $rating->getAttribute('aria-label') . ' : ' . strip_tags($summary->innertext); @@ -177,7 +185,32 @@ class KununuBridge extends BridgeAbstract { $description = $article->find('[itemprop=reviewBody]', 0) or returnServerError('Cannot find article description!'); - return $description->innertext; + $retVal = $description->innertext; + + if($this->getInput('include_ratings') + && ($ratings = $article->find('.review-ratings .rating-group'))) { + $retVal .= (empty($retVal) ? '' : '
') . ''; + foreach($ratings as $rating) { + $retVal .= << + +EOD; + } + $retVal .= '
{$rating->find('.rating-title', 0)->plaintext} + {$rating->find('.rating-badge', 0)->plaintext} +
'; + } + + if($this->getInput('include_benefits') + && ($benefits = $article->find('benefit'))) { + $retVal .= (empty($retVal) ? '' : '
') . '
    '; + foreach($benefits as $benefit) { + $retVal .= "
  • {$benefit->plaintext}
    • "; + } + $retVal .= '
'; + } + + return $retVal; } /** diff --git a/bridges/LaCentraleBridge.php b/bridges/LaCentraleBridge.php new file mode 100644 index 0000000..baaaa58 --- /dev/null +++ b/bridges/LaCentraleBridge.php @@ -0,0 +1,477 @@ + array( + 'name' => 'Type de véhicule', + 'type' => 'list', + 'values' => array( + 'Voiture' => 'car', + 'Camion/Pickup' => 'truck', + 'Moto' => 'moto', + 'Scooter' => 'scooter', + 'Quad' => 'quad', + 'Caravane/Camping-car' => 'mobileHome' + ) + ), + 'brand' => array( + 'name' => 'Marque', + 'type' => 'list', + 'values' => array( + '' => '', + 'ABARTH' => 'ABARTH', + 'AC' => 'AC', + 'AIXAM' => 'AIXAM', + 'ALFA ROMEO' => 'ALFA ROMEO', + 'ALKE' => 'ALKE', + 'ALPINA' => 'ALPINA', + 'ALPINE' => 'ALPINE', + 'AMC' => 'AMC', + 'ANAIG' => 'ANAIG', + 'APRILIA' => 'APRILIA', + 'ARIEL' => 'ARIEL', + 'ASTON MARTIN' => 'ASTON MARTIN', + 'AUDI' => 'AUDI', + 'AUSTIN HEALEY' => 'AUSTIN HEALEY', + 'AUSTIN' => 'AUSTIN', + 'AUTOBIANCHI' => 'AUTOBIANCHI', + 'AVINTON' => 'AVINTON', + 'BELLIER' => 'BELLIER', + 'BENELLI' => 'BENELLI', + 'BENTLEY' => 'BENTLEY', + 'BETA' => 'BETA', + 'BMW' => 'BMW', + 'BOLLORE' => 'BOLLORE', + 'BRIXTON' => 'BRIXTON', + 'BUELL' => 'BUELL', + 'BUGATTI' => 'BUGATTI', + 'BUICK' => 'BUICK', + 'BULLIT' => 'BULLIT', + 'CADILLAC' => 'CADILLAC', + 'CASALINI' => 'CASALINI', + 'CATERHAM' => 'CATERHAM', + 'CHATENET' => 'CHATENET', + 'CHEVROLET' => 'CHEVROLET', + 'CHRYSLER' => 'CHRYSLER', + 'CHUNLAN' => 'CHUNLAN', + 'CITROEN' => 'CITROEN', + 'COURB' => 'COURB', + 'CR&S' => 'CR&S', + 'CUPRA' => 'CUPRA', + 'CYCLONE' => 'CYCLONE', + 'DACIA' => 'DACIA', + 'DAELIM' => 'DAELIM', + 'DAEWOO' => 'DAEWOO', + 'DAF' => 'DAF', + 'DAIHATSU' => 'DAIHATSU', + 'DANGEL' => 'DANGEL', + 'DATSUN' => 'DATSUN', + 'DE SOTO' => 'DE SOTO', + 'DE TOMASO' => 'DE TOMASO', + 'DERBI' => 'DERBI', + 'DEVINCI' => 'DEVINCI', + 'DODGE' => 'DODGE', + 'DONKERVOORT' => 'DONKERVOORT', + 'DS' => 'DS', + 'DUCATI' => 'DUCATI', + 'DUCATY' => 'DUCATY', + 'DUE' => 'DUE', + 'ENFIELD' => 'ENFIELD', + 'EXCALIBUR' => 'EXCALIBUR', + 'FACEL VEGA' => 'FACEL VEGA', + 'FANTIC MOTOR' => 'FANTIC MOTOR', + 'FERRARI' => 'FERRARI', + 'FIAT' => 'FIAT', + 'FISKER' => 'FISKER', + 'FORD' => 'FORD', + 'FUSO' => 'FUSO', + 'GAS GAS' => 'GAS GAS', + 'GILERA' => 'GILERA', + 'GMC' => 'GMC', + 'GOWINN' => 'GOWINN', + 'GRANDIN' => 'GRANDIN', + 'HARLEY DAVIDSON' => 'HARLEY DAVIDSON', + 'HOMMELL' => 'HOMMELL', + 'HONDA' => 'HONDA', + 'HUMMER' => 'HUMMER', + 'HUSABERG' => 'HUSABERG', + 'HUSQVARNA' => 'HUSQVARNA', + 'HYOSUNG' => 'HYOSUNG', + 'HYUNDAI' => 'HYUNDAI', + 'INDIAN' => 'INDIAN', + 'INFINITI' => 'INFINITI', + 'INNOCENTI' => 'INNOCENTI', + 'ISUZU' => 'ISUZU', + 'IVECO' => 'IVECO', + 'JAGUAR' => 'JAGUAR', + 'JDM SIMPA' => 'JDM SIMPA', + 'JEEP' => 'JEEP', + 'JENSEN' => 'JENSEN', + 'JIAYUAN' => 'JIAYUAN', + 'KAWASAKI' => 'KAWASAKI', + 'KEEWAY' => 'KEEWAY', + 'KIA' => 'KIA', + 'KSR' => 'KSR', + 'KTM' => 'KTM', + 'KYMCO' => 'KYMCO', + 'LADA' => 'LADA', + 'LAMBORGHINI' => 'LAMBORGHINI', + 'LANCIA' => 'LANCIA', + 'LAND ROVER' => 'LAND ROVER', + 'LEXUS' => 'LEXUS', + 'LIGIER' => 'LIGIER', + 'LINCOLN' => 'LINCOLN', + 'LONDON TAXI COMPANY' => 'LONDON TAXI COMPANY', + 'LOTUS' => 'LOTUS', + 'MAGPOWER' => 'MAGPOWER', + 'MAN' => 'MAN', + 'MASAI' => 'MASAI', + 'MASERATI' => 'MASERATI', + 'MASH' => 'MASH', + 'MATRA' => 'MATRA', + 'MAYBACH' => 'MAYBACH', + 'MAZDA' => 'MAZDA', + 'MCLAREN' => 'MCLAREN', + 'MEGA' => 'MEGA', + 'MERCEDES' => 'MERCEDES', + 'MERCEDES-AMG' => 'MERCEDES-AMG', + 'MERCURY' => 'MERCURY', + 'MEYERS MANX' => 'MEYERS MANX', + 'MG' => 'MG', + 'MIA ELECTRIC' => 'MIA ELECTRIC', + 'MICROCAR' => 'MICROCAR', + 'MINAUTO' => 'MINAUTO', + 'MINI' => 'MINI', + 'MITSUBISHI' => 'MITSUBISHI', + 'MORGAN' => 'MORGAN', + 'MORRIS' => 'MORRIS', + 'MOTO GUZZI' => 'MOTO GUZZI', + 'MOTO MORINI' => 'MOTO MORINI', + 'MOTOBECANE' => 'MOTOBECANE', + 'MPM MOTORS' => 'MPM MOTORS', + 'MV AGUSTA' => 'MV AGUSTA', + 'NISSAN' => 'NISSAN', + 'NORTON' => 'NORTON', + 'NSU' => 'NSU', + 'OLDSMOBILE' => 'OLDSMOBILE', + 'OPEL' => 'OPEL', + 'ORCAL' => 'ORCAL', + 'OSSA' => 'OSSA', + 'PACKARD' => 'PACKARD', + 'PANTHER' => 'PANTHER', + 'PEUGEOT' => 'PEUGEOT', + 'PGO' => 'PGO', + 'PIAGGIO' => 'PIAGGIO', + 'PLYMOUTH' => 'PLYMOUTH', + 'POLARIS' => 'POLARIS', + 'PONTIAC' => 'PONTIAC', + 'PORSCHE' => 'PORSCHE', + 'REALM' => 'REALM', + 'REGAL RAPTOR' => 'REGAL RAPTOR', + 'RENAULT' => 'RENAULT', + 'RIEJU' => 'RIEJU', + 'ROLLS ROYCE' => 'ROLLS ROYCE', + 'ROVER' => 'ROVER', + 'ROYAL ENFIELD' => 'ROYAL ENFIELD', + 'SAAB' => 'SAAB', + 'SANTANA' => 'SANTANA', + 'SCANIA' => 'SCANIA', + 'SEAT' => 'SEAT', + 'SECMA' => 'SECMA', + 'SHELBY' => 'SHELBY', + 'SHERCO' => 'SHERCO', + 'SIMCA' => 'SIMCA', + 'SKODA' => 'SKODA', + 'SMART' => 'SMART', + 'SPYKER' => 'SPYKER', + 'SSANGYONG' => 'SSANGYONG', + 'STUDEBAKER' => 'STUDEBAKER', + 'SUBARU' => 'SUBARU', + 'SUNBEAM' => 'SUNBEAM', + 'SUZUKI' => 'SUZUKI', + 'SWM' => 'SWM', + 'SYM' => 'SYM', + 'TALBOT SIMCA' => 'TALBOT SIMCA', + 'TALBOT' => 'TALBOT', + 'TEILHOL' => 'TEILHOL', + 'TESLA' => 'TESLA', + 'TM' => 'TM', + 'TNT MOTOR' => 'TNT MOTOR', + 'TOYOTA' => 'TOYOTA', + 'TRIUMPH' => 'TRIUMPH', + 'TVR' => 'TVR', + 'VAUXHALL' => 'VAUXHALL', + 'VESPA' => 'VESPA', + 'VICTORY' => 'VICTORY', + 'VOLKSWAGEN' => 'VOLKSWAGEN', + 'VOLVO' => 'VOLVO', + 'VOXAN' => 'VOXAN', + 'WIESMANN' => 'WIESMANN', + 'YAMAHA' => 'YAMAHA', + 'YCF' => 'YCF', + 'ZERO' => 'ZERO', + 'ZONGSHEN' => 'ZONGSHEN' + ) + ), + 'model' => array( + 'name' => 'Modèle', + 'type' => 'text', + 'title' => 'Get the exact name on LaCentrale' + ), + 'versions' => array( + 'name' => 'Version(s)', + 'type' => 'text', + 'title' => 'Get the exact name(s) on LaCentrale. Separate by comma' + ), + 'category' => array( + 'name' => 'Catégorie', + 'type' => 'list', + 'values' => array( + '' => '', + 'Voiture' => array( + '4x4, SUV & Crossover' => '47', + 'Citadine' => '40', + 'Berline' => '41_42', + 'Break' => '43', + 'Cabriolet' => '46', + 'Coupé' => '45', + 'Monospace' => '44', + 'Bus et minibus' => '82', + 'Fourgonnette' => '85', + 'Fourgon (< 3,5 tonnes)' => '81', + 'Pick-up' => '50', + 'Voiture société, commerciale' => '80', + 'Sans permis' => '48', + 'Camion (> 3,5 tonnes)' => '83', + ), + 'Camion/Pickup' => array( + 'Camion (> 3,5 tonnes)' => '83', + 'Fourgon (< 3,5 tonnes)' => '81', + 'Bus et minibus' => '82', + 'Fourgonnette' => '85', + 'Pick-up' => '50', + 'Voiture société, commerciale' => '80' + ), + 'Moto' => array( + 'Custom' => '60', + 'Offroad' => '61', + 'Roadster' => '62', + 'GT' => '63', + 'Mini moto' => '64', + 'Mobylette' => '65', + 'Supermotard' => '66', + 'Trail' => '67', + 'Side-car' => '69', + 'Sportive' => '68' + ), + 'Caravane/Camping-car' => array( + 'Caravane' => '423', + 'Profilé' => '506', + 'Fourgon aménagé' => '507', + 'Intégral' => '508', + 'Capucine' => '510' + ) + ) + ), + 'pricemin' => array( + 'name' => 'Prix min', + 'type' => 'number' + ), + 'pricemax' => array( + 'name' => 'Prix max', + 'type' => 'number' + ), + 'location' => array( + 'name' => 'CP ou département', + 'type' => 'number', + 'title' => 'Only one' + ), + 'distance' => array( + 'name' => 'Rayon de recherche', + 'type' => 'list', + 'values' => array( + '' => '', + '10 km' => '1', + '20 km' => '2', + '50 km' => '3', + '100 km' => '4', + '200 km' => '5' + ) + ), + 'region' => array( + 'name' => 'Région', + 'type' => 'list', + 'values' => array( + '' => '', + 'Auvergne-Rhône-Alpes' => 'FR-ARA', + 'Bourgogne-Franche-Comté' => 'FR-BFC', + 'Bretagne' => 'FR-BRE', + 'Centre-Val de Loire' => 'FR-CVL', + 'Corse' => 'FR-COR', + 'Grand Est' => 'FR-GES', + 'Hauts-de-France' => 'FR-HDF', + 'Île-de-France' => 'FR-IDF', + 'Normandie' => 'FR-NOR', + 'Nouvelle-Aquitaine' => 'FR-PAC', + 'Occitanie' => 'FR-PDL', + 'Pays de la Loire' => 'FR-OCC', + 'Provence-Alpes-Côte d\'Azur' => 'FR-NAQ' + ) + ), + 'mileagemin' => array( + 'name' => 'Kilométrage min', + 'type' => 'number' + ), + 'mileagemax' => array( + 'name' => 'Kilométrage max', + 'type' => 'number' + ), + 'yearmin' => array( + 'name' => 'Année min', + 'type' => 'number' + ), + 'yearmax' => array( + 'name' => 'Année max', + 'type' => 'number' + ), + 'cubiccapacitymin' => array( + 'name' => 'Cylindrée min', + 'type' => 'number' + ), + 'cubiccapacitymax' => array( + 'name' => 'Cylindrée max', + 'type' => 'number' + ), + 'fuel' => array( + 'name' => 'Énergie', + 'type' => 'list', + 'values' => array( + '' => '', + 'Diesel' => 'dies', + 'Essence' => 'ess', + 'Électrique' => 'elec', + 'Hybride' => 'hyb', + 'GPL' => 'gpl', + 'Bioéthanol' => 'eth', + 'Autre' => 'alt' + ) + ), + 'gearbox' => array( + 'name' => 'Boite de vitesse', + 'type' => 'list', + 'values' => array( + '' => '', + 'Boite automatique' => 'AUTO', + 'Boite mécanique' => 'MANUAL' + ) + ), + 'doors' => array( + 'name' => 'Nombre de portes', + 'type' => 'list', + 'values' => array( + '' => '', + '2 portes' => '2', + '3 portes' => '3', + '4 portes' => '4', + '5 portes' => '5', + '6 portes ou plus' => '6' + ) + ), + 'firsthand' => array( + 'name' => 'Première main', + 'type' => 'checkbox' + ), + 'seller' => array( + 'name' => 'Vendeur', + 'type' => 'list', + 'values' => array( + '' => '', + 'Particulier' => 'PART', + 'Professionel' => 'PRO' + ) + ), + 'sort' => array( + 'name' => 'Tri', + 'type' => 'list', + 'values' => array( + 'Prix (croissant)' => 'priceAsc', + 'Prix (décroissant)' => 'priceDesc', + 'Marque (croissant)' => 'makeAsc', + 'Marque (décroissant)' => 'makeDesc', + 'Kilométrage (croissant)' => 'mileageAsc', + 'Kilométrage (décroissant)' => 'mileageDesc', + 'Année (croissant)' => 'yearAsc', + 'Année (décroissant)' => 'yearDesc', + 'Département (croissant)' => 'visitPlaceAsc', + 'Département (décroissant)' => 'visitPlaceDesc' + ) + ), + )); + + public function collectData(){ + // check data + if(!empty($this->getInput('distance')) + && is_null($this->getInput('location'))) { + returnClientError('You need a place ("CP ou département") to search arround.'); + } + + $params = array( + 'vertical' => $this->getInput('type'), + 'makesModelsCommercialNames' => $this->getInput('brand') . ':' . $this->getInput('model'), + 'versions' => $this->getInput('versions'), + 'categories' => $this->getInput('category'), + 'priceMin' => $this->getInput('pricemin'), + 'priceMax' => $this->getInput('pricemax'), + 'dptCp' => $this->getInput('location'), + 'distance' => $this->getInput('distance'), + 'regions' => $this->getInput('region'), + 'mileageMin' => $this->getInput('mileagemin'), + 'mileageMax' => $this->getInput('mileagemax'), + 'yearMin' => $this->getInput('yearmin'), + 'yearMax' => $this->getInput('yearmax'), + 'cubicMin' => $this->getInput('cubiccapacitymin'), + 'cubicMax' => $this->getInput('cubiccapacitymax'), + 'energies' => $this->getInput('fuel'), + 'firstHand' => $this->getInput('firsthand') ? 'true' : 'false', + 'gearbox' => $this->getInput('gearbox'), + 'doors' => $this->getInput('doors'), + 'sortBy' => $this->getInput('sort') + ); + $url = self::URI . 'listing?' . http_build_query($params); + $html = getSimpleHTMLDOM($url) + or returnServerError('Could not request LaCentrale.'); + + foreach($html->find('.linkAd') as $element) { + + $item = array(); + $item['uri'] = trim(self::URI, '/') . $element->href; + $item['title'] = $element->find('.brandModel', 0)->plaintext; + $item['sellerType'] = $element->find('.typeSeller', 0)->plaintext; + $item['author'] = $item['sellerType']; + $item['version'] = $element->find('.version', 0)->plaintext; + $item['price'] = $element->find('.fieldPrice', 0)->plaintext; + $item['year'] = $element->find('.fieldYear', 0)->plaintext; + $item['mileage'] = $element->find('.fieldMileage', 0)->plaintext; + $item['departement'] = str_replace(',', '', $element->find('.dptCont', 0)->plaintext); + $item['thumbnail'] = $element->find('.imgContent img', 0)->src; + $item['enclosures'] = array($item['thumbnail']); + + $item['content'] = ' + +
Variation : ' . $item['version'] + . '
Prix : ' . $item['price'] + . '
Année : ' . $item['year'] + . '
Kilométrage : ' . $item['mileage'] + . '
Département : ' . $item['departement'] + . '
Type de vendeur : ' . $item['sellerType']; + + $this->items[] = $item; + + } + } +} diff --git a/bridges/LeBonCoinBridge.php b/bridges/LeBonCoinBridge.php index 36196cb..519fc91 100644 --- a/bridges/LeBonCoinBridge.php +++ b/bridges/LeBonCoinBridge.php @@ -356,6 +356,7 @@ class LeBonCoinBridge extends BridgeAbstract { $data = $this->buildRequestJson(); $header = array( + 'User-Agent: LBC;Android;Null;Null;Null;Null;Null;Null;Null;Null', 'Content-Type: application/json', 'Content-Length: ' . strlen($data), 'api_key: ' . self::$LBC_API_KEY diff --git a/bridges/LeMondeInformatiqueBridge.php b/bridges/LeMondeInformatiqueBridge.php index 09bcf6a..45aa607 100644 --- a/bridges/LeMondeInformatiqueBridge.php +++ b/bridges/LeMondeInformatiqueBridge.php @@ -20,12 +20,13 @@ class LeMondeInformatiqueBridge extends FeedExpander { str_replace( '/grande/', '/petite/', - $article_html->find('.article-image', 0)->find('img', 0)->src + $article_html->find('.article-image > img, figure > img', 0)->src ) ); //No response header sets the encoding, explicit conversion is needed or subsequent xml_encode() will fail - $item['content'] = utf8_encode($this->cleanArticle($article_html->find('div.col-primary', 0)->innertext)); + $content_node = $article_html->find('div.col-primary, div.col-sm-9', 0); + $item['content'] = utf8_encode($this->cleanArticle($content_node->innertext)); $item['author'] = utf8_encode($article_html->find('div.author-infos', 0)->find('b', 0)->plaintext); return $item; diff --git a/bridges/MangareaderBridge.php b/bridges/MangareaderBridge.php index 9153706..9ecb0fe 100644 --- a/bridges/MangareaderBridge.php +++ b/bridges/MangareaderBridge.php @@ -13,7 +13,6 @@ class MangareaderBridge extends BridgeAbstract { 'category' => array( 'name' => 'Category', 'type' => 'list', - 'required' => true, 'values' => array( 'All' => 'all', 'Action' => 'action', diff --git a/bridges/MastodonBridge.php b/bridges/MastodonBridge.php new file mode 100644 index 0000000..9e131b7 --- /dev/null +++ b/bridges/MastodonBridge.php @@ -0,0 +1,89 @@ + array( + 'name' => 'Canonical username (ex : @sebsauvage@framapiaf.org)', + 'required' => true, + ), + 'norep' => array( + 'name' => 'Without replies', + 'type' => 'checkbox', + 'title' => 'Only return initial toots' + ), + 'noboost' => array( + 'name' => 'Without boosts', + 'required' => false, + 'type' => 'checkbox', + 'title' => 'Hide boosts' + ) + )); + + public function getName(){ + switch($this->queriedContext) { + case 'By username': + return $this->getInput('canusername'); + default: return parent::getName(); + } + } + + protected function parseItem($newItem){ + $item = parent::parseItem($newItem); + + $content = str_get_html($item['content']); + $title = str_get_html($item['title']); + + $item['title'] = $content->plaintext; + + if(strlen($item['title']) > 75) { + $item['title'] = substr($item['title'], 0, strpos(wordwrap($item['title'], 75), "\n")) . '...'; + } + + if(strpos($title, 'shared a status by') !== false) { + if($this->getInput('noboost')) { + return null; + } + + preg_match('/shared a status by (\S{0,})/', $title, $matches); + $item['title'] = 'Boost ' . $matches[1] . ' ' . $item['title']; + $item['author'] = $matches[1]; + } else { + $item['author'] = $this->getInput('canusername'); + } + + // Check if it's a initial toot or a response + if($this->getInput('norep') && preg_match('/^@.+/', trim($content->plaintext))) { + return null; + } + + return $item; + } + + private function getInstance(){ + preg_match('/^@[a-zA-Z0-9_]+@(.+)/', $this->getInput('canusername'), $matches); + return $matches[1]; + } + + private function getUsername(){ + preg_match('/^@([a-zA-Z_0-9_]+)@.+/', $this->getInput('canusername'), $matches); + return $matches[1]; + } + + public function getURI(){ + if($this->getInput('canusername')) + return 'https://' . $this->getInstance() . '/users/' . $this->getUsername() . '.atom'; + + return parent::getURI(); + } + + public function collectData(){ + return $this->collectExpandableDatas($this->getURI()); + } +} diff --git a/bridges/MediapartBridge.php b/bridges/MediapartBridge.php new file mode 100644 index 0000000..15d1d3e --- /dev/null +++ b/bridges/MediapartBridge.php @@ -0,0 +1,60 @@ + array( + 'name' => 'Single page article', + 'type' => 'checkbox', + 'title' => 'Display long articles on a single page', + 'defaultValue' => 'checked' + ), + 'mpsessid' => array( + 'name' => 'MPSESSID', + 'type' => 'text', + 'title' => 'Value of the session cookie MPSESSID' + ) + ) + ); + const CACHE_TIMEOUT = 7200; // 2h + const DESCRIPTION = 'Returns the newest articles.'; + + public function collectData() { + $url = self::URI . 'articles/feed'; + $this->collectExpandableDatas($url); + } + + protected function parseItem($newsItem) { + $item = parent::parseItem($newsItem); + + // Enable single page mode? + if ($this->getInput('single_page_mode') === true) { + $item['uri'] .= '?onglet=full'; + } + + // If a session cookie is defined, get the full article + $mpsessid = $this->getInput('mpsessid'); + if (!empty($mpsessid)) { + // Set the session cookie + $opt = array(); + $opt[CURLOPT_COOKIE] = 'MPSESSID=' . $mpsessid; + + // Get the page + $articlePage = getSimpleHTMLDOM( + $newsItem->link . '?onglet=full', + array(), + $opt); + + // Extract the article content + $content = $articlePage->find('div.content-article', 0)->innertext; + $content = sanitize($content); + $content = defaultLinkTo($content, static::URI); + $item['content'] .= $content; + } + + return $item; + } +} diff --git a/bridges/MozillaBugTrackerBridge.php b/bridges/MozillaBugTrackerBridge.php new file mode 100644 index 0000000..356bedc --- /dev/null +++ b/bridges/MozillaBugTrackerBridge.php @@ -0,0 +1,153 @@ + array( + 'id' => array( + 'name' => 'Bug tracking ID', + 'type' => 'number', + 'required' => true, + 'title' => 'Insert bug tracking ID', + 'exampleValue' => 121241 + ), + 'limit' => array( + 'name' => 'Number of comments to return', + 'type' => 'number', + 'required' => false, + 'title' => 'Specify number of comments to return', + 'defaultValue' => -1 + ), + 'sorting' => array( + 'name' => 'Sorting', + 'type' => 'list', + 'required' => false, + 'title' => 'Defines the sorting order of the comments returned', + 'defaultValue' => 'of', + 'values' => array( + 'Oldest first' => 'of', + 'Latest first' => 'lf' + ) + ) + ) + ); + + private $bugid = ''; + private $bugdesc = ''; + + public function getIcon() { + return self::URI . '/extensions/BMO/web/images/favicon.ico'; + } + + public function collectData(){ + $limit = $this->getInput('limit'); + $sorting = $this->getInput('sorting'); + + // We use the print preview page for simplicity + $html = getSimpleHTMLDOMCached($this->getURI() . '&format=multiple', + 86400, + null, + null, + true, + true, + DEFAULT_TARGET_CHARSET, + false, // Do NOT remove line breaks + DEFAULT_BR_TEXT, + DEFAULT_SPAN_TEXT); + + if($html === false) + returnServerError('Failed to load page!'); + + // Store header information into private members + $this->bugid = $html->find('#bugzilla-body', 0)->find('a', 0)->innertext; + $this->bugdesc = $html->find('table.bugfields', 0)->find('tr', 0)->find('td', 0)->innertext; + + // Get and limit comments + $comments = $html->find('.bz_comment_table div.bz_comment'); + + if($limit > 0 && count($comments) > $limit) { + $comments = array_slice($comments, count($comments) - $limit, $limit); + } + + // Order comments + switch($sorting) { + case 'lf': $comments = array_reverse($comments, true); + case 'of': + default: // Nothing to do, keep original order + } + + foreach($comments as $comment) { + $comment = $this->inlineStyles($comment); + + $item = array(); + $item['uri'] = $this->getURI() . '#' . $comment->id; + $item['author'] = $comment->find('span.bz_comment_user', 0)->innertext; + $item['title'] = $comment->find('span.bz_comment_number', 0)->find('a', 0)->innertext; + $item['timestamp'] = strtotime($comment->find('span.bz_comment_time', 0)->innertext); + $item['content'] = $comment->find('pre.bz_comment_text', 0)->innertext; + + // Fix line breaks (they use LF) + $item['content'] = str_replace("\n", '
', $item['content']); + + // Fix relative URIs + $item['content'] = $this->replaceRelativeURI($item['content']); + + $this->items[] = $item; + } + + } + + public function getURI(){ + switch($this->queriedContext) { + case 'Bug comments': + return parent::getURI() + . '/show_bug.cgi?id=' + . $this->getInput('id'); + break; + default: return parent::getURI(); + } + } + + public function getName(){ + switch($this->queriedContext) { + case 'Bug comments': + return 'Bug ' + . $this->bugid + . ' tracker for ' + . $this->bugdesc + . ' - ' + . parent::getName(); + break; + default: return parent::getName(); + } + } + + /** + * Replaces all relative URIs with absolute ones + * + * @param string $content The source string + * @return string Returns the source string with all relative URIs replaced + * by absolute ones. + */ + private function replaceRelativeURI($content){ + return preg_replace('/href="(?!http)/', 'href="' . self::URI . '/', $content); + } + + /** + * Adds styles as attributes to tags with known classes + * + * @param object $html A simplehtmldom object + * @return object Returns the original object with styles added as + * attributes. + */ + private function inlineStyles($html){ + foreach($html->find('.bz_obsolete') as $element) { + $element->style = 'text-decoration:line-through;'; + } + + return $html; + } +} diff --git a/bridges/MozillaSecurityBridge.php b/bridges/MozillaSecurityBridge.php index 0b951a1..52672f5 100644 --- a/bridges/MozillaSecurityBridge.php +++ b/bridges/MozillaSecurityBridge.php @@ -21,7 +21,8 @@ class MozillaSecurityBridge extends BridgeAbstract { $item['title'] = $element->innertext; $item['timestamp'] = strtotime($element->innertext); $item['content'] = $element->next_sibling()->innertext; - $item['uri'] = self::URI; + $item['uri'] = self::URI . '?' . $item['timestamp']; + $item['uid'] = self::URI . '?' . $item['timestamp']; $this->items[] = $item; } } diff --git a/bridges/MydealsBridge.php b/bridges/MydealsBridge.php index aa5bb48..603f4e0 100644 --- a/bridges/MydealsBridge.php +++ b/bridges/MydealsBridge.php @@ -17,13 +17,11 @@ class MydealsBridge extends PepperBridgeAbstract { 'hide_expired' => array( 'name' => 'Abgelaufenes ausblenden', 'type' => 'checkbox', - 'required' => true ), 'hide_local' => array( 'name' => 'Lokales ausblenden', 'type' => 'checkbox', 'title' => 'Deals im physischen Geschäft ausblenden', - 'required' => true ), 'priceFrom' => array( 'name' => 'Minimaler Preis', @@ -43,7 +41,6 @@ class MydealsBridge extends PepperBridgeAbstract { 'group' => array( 'name' => 'Gruppen', 'type' => 'list', - 'required' => true, 'title' => 'Gruppe, deren Deals angezeigt werden müssen', 'values' => array( 'Elektronik' => 'elektronik', @@ -66,7 +63,6 @@ class MydealsBridge extends PepperBridgeAbstract { 'order' => array( 'name' => 'sortieren nach', 'type' => 'list', - 'required' => true, 'title' => 'Sortierung der deals', 'values' => array( 'Vom heißesten zum kältesten Deal' => '', diff --git a/bridges/NYTBridge.php b/bridges/NYTBridge.php new file mode 100644 index 0000000..687d088 --- /dev/null +++ b/bridges/NYTBridge.php @@ -0,0 +1,26 @@ +collectExpandableDatas('https://rss.nytimes.com/services/xml/rss/nyt/HomePage.xml', 15); + } + + protected function parseItem($newsItem){ + $item = parent::parseItem($newsItem); + // $articlePage gets the entire page's contents + $articlePage = getSimpleHTMLDOM($newsItem->link); + // figure contain's the main article image + $article = $articlePage->find('figure', 0); + // p > css-exrw3m has the actual article + foreach($articlePage->find('p.css-exrw3m') as $element) + $article = $article . $element; + $item['content'] = $article; + return $item; + } +} diff --git a/bridges/NationalGeographicBridge.php b/bridges/NationalGeographicBridge.php new file mode 100644 index 0000000..dfccd25 --- /dev/null +++ b/bridges/NationalGeographicBridge.php @@ -0,0 +1,194 @@ + array( + self::PARAMETER_TOPIC => array( + 'name' => 'Topic', + 'type' => 'list', + 'values' => array( + self::TOPIC_MAGAZINE => 'magazine', + self::TOPIC_LATEST_STORIES => 'latest-stories' + ), + 'title' => 'Select your topic', + 'defaultValue' => 'Magazine' + ) + ), + 'global' => array( + self::PARAMETER_FULL_ARTICLE => array( + 'name' => 'Full Article', + 'type' => 'checkbox', + 'title' => 'Enable to load full articles (takes longer)' + ) + ) + ); + + private $topicName = ''; + + public function getURI() { + switch ($this->queriedContext) { + case self::CONTEXT_BY_TOPIC: { + return self::URI . $this->getInput(self::PARAMETER_TOPIC); + } break; + default: { + return parent::getURI(); + } + } + } + + public function collectData() { + $this->topicName = $this->getTopicName($this->getInput(self::PARAMETER_TOPIC)); + + switch($this->topicName) { + case self::TOPIC_MAGAZINE: { + return $this->collectMagazine(); + } break; + case self::TOPIC_LATEST_STORIES: { + return $this->collectLatestStories(); + } break; + default: { + returnServerError('Unknown topic: "' . $this->topicName . '"'); + } + } + } + + public function getName() { + switch ($this->queriedContext) { + case self::CONTEXT_BY_TOPIC: { + return static::NAME . ': ' . $this->topicName; + } break; + default: { + return parent::getName(); + } + } + } + + private function getTopicName($topic) { + return array_search($topic, static::PARAMETERS[self::CONTEXT_BY_TOPIC][self::PARAMETER_TOPIC]['values']); + } + + private function collectMagazine() { + $uri = $this->getURI(); + + $html = getSimpleHTMLDOM($uri) + or returnServerError('Could not request ' . $uri); + + $script = $html->find('#lead-component script')[0]; + + $json = json_decode($script->innertext, true); + + // This is probably going to break in the future, fix it then :) + foreach($json['body']['0']['multilayout_promo_beta']['stories'] as $story) { + $this->addStory($story); + } + } + + private function collectLatestStories() { + $uri = self::URI . 'latest-stories/_jcr_content/content/hubfeed.promo-hub-feed-all-stories.json'; + + $json_raw = getContents($uri) + or returnServerError('Could not request ' . $uri); + + foreach(json_decode($json_raw, true) as $story) { + $this->addStory($story); + } + } + + private function addStory($story) { + $title = 'Unknown title'; + $content = ''; + + foreach($story['components'] as $component) { + switch($component['content_type']) { + case 'title': { + $title = $component['title']['text']; + } break; + case 'dek': { + $content = $component['dek']['text']; + } break; + } + } + + $item = array(); + + $item['uri'] = $story['uri']; + $item['title'] = $title; + + // if full article is requested! + if ($this->getInput(self::PARAMETER_FULL_ARTICLE)) + $item['content'] = $this->getFullArticle($item['uri']); + else + $item['content'] = $content; + + if (isset($story['promo_image'])) { + switch($story['promo_image']['content_type']) { + case 'image': { + $item['enclosures'][] = $story['promo_image']['image']['uri']; + } break; + } + } + + if (isset($story['lead_media'])) { + $media = $story['lead_media']; + switch($media['content_type']) { + case 'image': { + // Don't add if promo_image was added + if (empty($item['enclosures'])) + $item['enclosures'][] = $media['image']['uri']; + } break; + case 'image_gallery': { + foreach($media['image_gallery']['images'] as $image) { + $item['enclosures'][] = $image['uri']; + } + } break; + } + } + + $this->items[] = $item; + } + + private function getFullArticle($uri) { + $html = getSimpleHTMLDOMCached($uri) + or returnServerError('Could not load ' . $uri); + + $html = defaultLinkTo($html, $uri); + + $content = ''; + + foreach($html->find(' + .content > .smartbody.text, + .content > .section.image script[type="text/json"], + .content > .section.image span[itemprop="caption"], + .content > .section.inline script[type="text/json"] + ') as $element) { + if ($element->tag === 'script') { + $json = json_decode($element->innertext, true); + if (isset($json['src'])) { + $content .= '' . $json['alt'] . ''; + } elseif (isset($json['galleryType']) && isset($json['endpoint'])) { + $doc = getContents($json['endpoint']) + or returnServerError('Could not load ' . $json['endpoint']); + $json = json_decode($doc, true); + foreach($json['items'] as $item) { + $content .= '

' . $item['caption'] . '

'; + $content .= '' . $item['caption'] . ''; + } + } + } else { + $content .= $element->outertext; + } + } + + return $content; + } +} diff --git a/bridges/NineGagBridge.php b/bridges/NineGagBridge.php index f526135..e726c73 100644 --- a/bridges/NineGagBridge.php +++ b/bridges/NineGagBridge.php @@ -11,7 +11,6 @@ class NineGagBridge extends BridgeAbstract { 'd' => array( 'name' => 'Section', 'type' => 'list', - 'required' => true, 'values' => array( 'Hot' => 'hot', 'Trending' => 'trending', @@ -28,7 +27,6 @@ class NineGagBridge extends BridgeAbstract { 'g' => array( 'name' => 'Section', 'type' => 'list', - 'required' => true, 'values' => array( 'Animals' => 'cute', 'Anime & Manga' => 'anime-manga', @@ -88,7 +86,6 @@ class NineGagBridge extends BridgeAbstract { 't' => array( 'name' => 'Type', 'type' => 'list', - 'required' => true, 'values' => array( 'Hot' => 'hot', 'Fresh' => 'fresh', diff --git a/bridges/NotAlwaysBridge.php b/bridges/NotAlwaysBridge.php index b2f4c35..c7758c3 100644 --- a/bridges/NotAlwaysBridge.php +++ b/bridges/NotAlwaysBridge.php @@ -21,8 +21,7 @@ class NotAlwaysBridge extends BridgeAbstract { 'Friendly' => 'friendly', 'Hopeless' => 'hopeless', 'Unfiltered' => 'unfiltered' - ), - 'required' => true + ) ) )); diff --git a/bridges/NovelUpdatesBridge.php b/bridges/NovelUpdatesBridge.php index 729eb48..05acd8e 100644 --- a/bridges/NovelUpdatesBridge.php +++ b/bridges/NovelUpdatesBridge.php @@ -3,7 +3,7 @@ class NovelUpdatesBridge extends BridgeAbstract { const MAINTAINER = 'albirew'; const NAME = 'Novel Updates'; - const URI = 'http://www.novelupdates.com/'; + const URI = 'https://www.novelupdates.com/'; const CACHE_TIMEOUT = 21600; // 6h const DESCRIPTION = 'Returns releases from Novel Updates'; const PARAMETERS = array( array( diff --git a/bridges/OnVaSortirBridge.php b/bridges/OnVaSortirBridge.php index ee6baf1..ed1dcb6 100644 --- a/bridges/OnVaSortirBridge.php +++ b/bridges/OnVaSortirBridge.php @@ -9,7 +9,6 @@ class OnVaSortirBridge extends FeedExpander { 'city' => array( 'name' => 'City', 'type' => 'list', - 'required' => true, 'values' => array( 'Agen' => 'Agen', 'Ajaccio' => 'Ajaccio', diff --git a/bridges/OneFortuneADayBridge.php b/bridges/OneFortuneADayBridge.php index ed0b5ec..62fe767 100644 --- a/bridges/OneFortuneADayBridge.php +++ b/bridges/OneFortuneADayBridge.php @@ -35,25 +35,33 @@ class OneFortuneADayBridge extends BridgeAbstract { '23:00' => 23, ), 'defaultValue' => 5 + ), + 'lucky' => array( + 'name' => 'Lucky number (optional)', + 'type' => 'text' ) )); const LIMIT_ITEMS = 7; const DAY_SECS = 86400; + public function getDescription(){ + return self::DESCRIPTION . '
Set a lucky number to get your personal quotes, like ' . mt_rand(); + } + public function collectData() { $time = gmmktime((int)$this->getInput('time'), 0, 0); if ($time > time()) $time -= self::DAY_SECS; for ($i = self::LIMIT_ITEMS; $i > 0; --$i) { - $seed = date('Ymd', $time); + $seed = gmdate('Ymd', $time) . $this->getInput('lucky'); $quote = $this->getQuote($seed); $item['title'] = strftime('%A, %x', $time); $item['content'] = htmlentities($quote, ENT_QUOTES, 'UTF-8'); $item['timestamp'] = $time; - $item['uri'] = 'urn:sha1:' . hash('sha1', $seed); + $item['uid'] = hash('sha1', $seed); $this->items[] = $item; diff --git a/bridges/OpenClassroomsBridge.php b/bridges/OpenClassroomsBridge.php index 5f0daca..4db7bc1 100644 --- a/bridges/OpenClassroomsBridge.php +++ b/bridges/OpenClassroomsBridge.php @@ -11,7 +11,6 @@ class OpenClassroomsBridge extends BridgeAbstract { 'u' => array( 'name' => 'Catégorie', 'type' => 'list', - 'required' => true, 'values' => array( 'Arts & Culture' => 'arts', 'Code' => 'code', diff --git a/bridges/PatreonBridge.php b/bridges/PatreonBridge.php new file mode 100644 index 0000000..57727a3 --- /dev/null +++ b/bridges/PatreonBridge.php @@ -0,0 +1,203 @@ + array( + 'name' => 'Creator', + 'type' => 'text', + 'required' => true, + 'title' => 'Creator name as seen in their page URL' + ) + )); + + public function collectData(){ + $html = getSimpleHTMLDOMCached($this->getURI(), 86400) + or returnServerError('Failed to load creator page at ' . $this->getURI()); + $regex = '#/api/campaigns/([0-9]+)#'; + if(preg_match($regex, $html->save(), $matches) > 0) { + $campaign_id = $matches[1]; + } else { + returnServerError('Could not find campaign ID'); + } + + $query = array( + 'include' => implode(',', array( + 'user', + 'attachments', + 'user_defined_tags', + //'campaign', + //'poll.choices', + //'poll.current_user_responses.user', + //'poll.current_user_responses.choice', + //'poll.current_user_responses.poll', + //'access_rules.tier.null', + //'images.null', + //'audio.null' + )), + 'fields' => array( + 'post' => implode(',', array( + //'change_visibility_at', + //'comment_count', + 'content', + //'current_user_can_delete', + //'current_user_can_view', + //'current_user_has_liked', + //'embed', + 'image', + //'is_paid', + //'like_count', + //'min_cents_pledged_to_view', + //'patreon_url', + //'patron_count', + //'pledge_url', + //'post_file', + //'post_metadata', + //'post_type', + 'published_at', + 'teaser_text', + //'thumbnail_url', + 'title', + //'upgrade_url', + 'url', + //'was_posted_by_campaign_owner' + )), + 'user' => implode(',', array( + //'image_url', + 'full_name', + //'url' + )) + ), + 'filter' => array( + 'contains_exclusive_posts' => true, + 'is_draft' => false, + 'campaign_id' => $campaign_id + ), + 'sort' => '-published_at' + ); + $posts = $this->apiGet('posts', $query); + + foreach($posts->data as $post) { + $item = array( + 'uri' => $post->attributes->url, + 'title' => $post->attributes->title, + 'timestamp' => $post->attributes->published_at, + 'content' => '', + 'uid' => 'patreon.com/' . $post->id + ); + + $user = $this->findInclude($posts, + 'user', + $post->relationships->user->data->id); + $item['author'] = $user->full_name; + + if(isset($post->attributes->image)) + $item['content'] .= '

'; + + if(isset($post->attributes->content)) { + $item['content'] .= $post->attributes->content; + } elseif (isset($post->attributes->teaser_text)) { + $item['content'] .= '

' + . $post->attributes->teaser_text + . '

'; + } + + if(isset($post->relationships->user_defined_tags)) { + $item['categories'] = array(); + foreach($post->relationships->user_defined_tags->data as $tag) { + $attrs = $this->findInclude($posts, 'post_tag', $tag->id); + $item['categories'][] = $attrs->value; + } + } + + if(isset($post->relationships->attachments)) { + $item['enclosures'] = array(); + foreach($post->relationships->attachments->data as $attachment) { + $attrs = $this->findInclude($posts, 'attachment', $attachment->id); + $item['enclosures'][] = $attrs->url; + } + } + + $this->items[] = $item; + } + } + + /* + * Searches the "included" array in an API response and returns attributes + * for the first match. + */ + private function findInclude($data, $type, $id) { + foreach($data->included as $include) + if($include->type === $type && $include->id === $id) + return $include->attributes; + } + + private function apiGet($endpoint, $query_data = array()) { + $query_data['json-api-version'] = 1.0; + $query_data['json-api-use-default-includes'] = 0; + + $url = 'https://www.patreon.com/api/' + . $endpoint + . '?' + . http_build_query($query_data); + + /* + * Accept-Language header and the CURL cipher list are for bypassing the + * Cloudflare anti-bot protection on the Patreon API. If this ever breaks, + * here are some other project that also deal with this: + * https://github.com/mikf/gallery-dl/issues/342 + * https://github.com/daemionfox/patreon-feed/issues/7 + * https://www.patreondevelopers.com/t/api-returning-cloudflare-challenge/2025 + * https://github.com/splitbrain/patreon-rss/issues/4 + */ + $header = array( + 'Accept-Language: en-US', + 'Content-Type: application/json' + ); + $opts = array( + CURLOPT_SSL_CIPHER_LIST => implode(':', array( + 'DEFAULT', + '!DHE-RSA-CHACHA20-POLY1305' + )) + ); + + $data = json_decode(getContents($url, $header, $opts)) + or returnServerError('API request to "' . $url . '" failed.'); + + return $data; + } + + public function getName(){ + if(!is_null($this->getInput('creator'))) + return $this->getInput('creator') . ' posts'; + + return parent::getName(); + } + + public function getURI(){ + if(!is_null($this->getInput('creator'))) + return self::URI . $this->getInput('creator'); + + return parent::getURI(); + } + + public function detectParameters($url){ + $params = array(); + + // Matches e.g. https://www.patreon.com/SomeCreator + $regex = '/^(https?:\/\/)?(www\.)?patreon\.com\/([^\/&?\n]+)/'; + if(preg_match($regex, $url, $matches) > 0) { + $params['creator'] = urldecode($matches[3]); + return $params; + } + + return null; + } +} diff --git a/bridges/PikabuBridge.php b/bridges/PikabuBridge.php index af603ac..987070d 100644 --- a/bridges/PikabuBridge.php +++ b/bridges/PikabuBridge.php @@ -6,6 +6,16 @@ class PikabuBridge extends BridgeAbstract { const DESCRIPTION = 'Выводит посты по тегу'; const MAINTAINER = 'em92'; + const PARAMETERS_FILTER = array( + 'name' => 'Фильтр', + 'type' => 'list', + 'values' => array( + 'Горячее' => 'hot', + 'Свежее' => 'new', + ), + 'defaultValue' => 'hot' + ); + const PARAMETERS = array( 'По тегу' => array( 'tag' => array( @@ -13,21 +23,38 @@ class PikabuBridge extends BridgeAbstract { 'exampleValue' => 'it', 'required' => true ), - 'filter' => array( - 'name' => 'Фильтр', - 'type' => 'list', - 'values' => array( - 'Горячее' => 'hot', - 'Свежее' => 'new', - ), - 'defaultValue' => 'hot' + 'filter' => self::PARAMETERS_FILTER + ), + 'По сообществу' => array( + 'community' => array( + 'name' => 'Сообщество', + 'exampleValue' => 'linux', + 'required' => true + ), + 'filter' => self::PARAMETERS_FILTER + ), + 'По пользователю' => array( + 'user' => array( + 'name' => 'Пользователь', + 'exampleValue' => 'admin', + 'required' => true ) ) ); + protected $title = null; + public function getURI() { if ($this->getInput('tag')) { return self::URI . '/tag/' . rawurlencode($this->getInput('tag')) . '/' . rawurlencode($this->getInput('filter')); + } else if ($this->getInput('user')) { + return self::URI . '/@' . rawurlencode($this->getInput('user')); + } else if ($this->getInput('community')) { + $uri = self::URI . '/community/' . rawurlencode($this->getInput('community')); + if ($this->getInput('filter') != 'hot') { + $uri .= '/' . rawurlencode($this->getInput('filter')); + } + return $uri; } else { return parent::getURI(); } @@ -38,10 +65,10 @@ class PikabuBridge extends BridgeAbstract { } public function getName() { - if (is_string($this->getInput('tag'))) { - return $this->getInput('tag') . ' - ' . parent::getName(); - } else { + if (is_null($this->title)) { return parent::getName(); + } else { + return $this->title . ' - ' . parent::getName(); } } @@ -52,6 +79,8 @@ class PikabuBridge extends BridgeAbstract { $text_html = iconv('windows-1251', 'utf-8', $text_html); $html = str_get_html($text_html); + $this->title = $html->find('title', 0)->innertext; + foreach($html->find('article.story') as $post) { $time = $post->find('time.story__datetime', 0); if (is_null($time)) continue; @@ -67,6 +96,11 @@ class PikabuBridge extends BridgeAbstract { } } + foreach($post->find('[data-type=gifx]') as $el) { + $src = $el->getAttribute('data-source'); + $el->outertext = ''; + } + foreach($post->find('img') as $img) { $src = $img->getAttribute('src'); if (!$src) { diff --git a/bridges/PinterestBridge.php b/bridges/PinterestBridge.php index 2917b26..3e51863 100644 --- a/bridges/PinterestBridge.php +++ b/bridges/PinterestBridge.php @@ -16,12 +16,6 @@ class PinterestBridge extends FeedExpander { 'name' => 'board', 'required' => true ) - ), - 'From search' => array( - 'q' => array( - 'name' => 'Keyword', - 'required' => true - ) ) ); @@ -29,17 +23,9 @@ class PinterestBridge extends FeedExpander { return 'https://s.pinimg.com/webapp/style/images/favicon-9f8f9adf.png'; } - public function collectData(){ - switch($this->queriedContext) { - case 'By username and board': - $this->collectExpandableDatas($this->getURI() . '.rss'); - $this->fixLowRes(); - break; - case 'From search': - default: - $html = getSimpleHTMLDOMCached($this->getURI()); - $this->getSearchResults($html); - } + public function collectData() { + $this->collectExpandableDatas($this->getURI() . '.rss'); + $this->fixLowRes(); } private function fixLowRes() { @@ -55,71 +41,21 @@ class PinterestBridge extends FeedExpander { } - private function getSearchResults($html){ - $json = json_decode($html->find('#jsInit1', 0)->innertext, true); - $results = $json['resourceDataCache'][0]['data']['results']; - - foreach($results as $result) { - $item = array(); - - $item['uri'] = self::URI . $result['board']['url']; - - // Some use regular titles, others provide 'advanced' infos, a few - // provide even less info. Thus we attempt multiple options. - $item['title'] = trim($result['title']); - - if($item['title'] === '') - $item['title'] = trim($result['rich_summary']['display_name']); + public function getURI() { - if($item['title'] === '') - $item['title'] = trim($result['grid_description']); - - $item['timestamp'] = strtotime($result['created_at']); - $item['username'] = $result['pinner']['username']; - $item['fullname'] = $result['pinner']['full_name']; - $item['avatar'] = $result['pinner']['image_small_url']; - $item['author'] = $item['username'] . ' (' . $item['fullname'] . ')'; - $item['content'] = '

' - . $item['username'] - . '
' - . $item['fullname'] - . '



' - . $result['description'] - . '

'; - - $item['enclosures'] = array($result['images']['orig']['url']); - - $this->items[] = $item; + if ($this->queriedContext === 'By username and board') { + return self::URI . '/' . urlencode($this->getInput('u')) . '/' . urlencode($this->getInput('b')); } - } - public function getURI(){ - switch($this->queriedContext) { - case 'By username and board': - $uri = self::URI . '/' . urlencode($this->getInput('u')) . '/' . urlencode($this->getInput('b'));// . '.rss'; - break; - case 'From search': - $uri = self::URI . '/search/?q=' . urlencode($this->getInput('q')); - break; - default: return parent::getURI(); - } - return $uri; + return parent::getURI(); } - public function getName(){ - switch($this->queriedContext) { - case 'By username and board': - $specific = $this->getInput('u') . ' - ' . $this->getInput('b'); - break; - case 'From search': - $specific = $this->getInput('q'); - break; - default: return parent::getName(); + public function getName() { + + if ($this->queriedContext === 'By username and board') { + return $this->getInput('u') . ' - ' . $this->getInput('b') . ' - ' . self::NAME; } - return $specific . ' - ' . self::NAME; + + return parent::getName(); } } diff --git a/bridges/PirateCommunityBridge.php b/bridges/PirateCommunityBridge.php new file mode 100644 index 0000000..fcf97b9 --- /dev/null +++ b/bridges/PirateCommunityBridge.php @@ -0,0 +1,88 @@ + array( + 'name' => 'Topic ID', + 'type' => 'number', + 'title' => 'Topic ID from topic URL. If the URL contains t=12 the ID is 12.', + 'required' => true + ))); + + private $feedName = ''; + + public function detectParameters($url){ + $parsed_url = parse_url($url); + + if($parsed_url['host'] !== 'raymanpc.com') + return null; + + parse_str($parsed_url['query'], $parsed_query); + + if($parsed_url['path'] === '/forum/viewtopic.php' + && array_key_exists('t', $parsed_query)) { + return array('t' => $parsed_query['t']); + } + + return null; + } + + public function getName() { + if(!empty($this->feedName)) + return $this->feedName; + + return parent::getName(); + } + + public function getURI(){ + if(!is_null($this->getInput('t'))) { + return self::URI + . 'forum/viewtopic.php?t=' + . $this->getInput('t') + . '&sd=d'; // sort posts decending by ate so first page has latest posts + } + + return parent::getURI(); + } + + public function collectData(){ + $html = getSimpleHTMLDOM($this->getURI()) + or returnServerError('Could not retrieve topic page at ' . $this->getURI()); + + $this->feedName = $html->find('head title', 0)->plaintext; + + foreach($html->find('.post') as $reply) { + $item = array(); + + $item['uri'] = $this->getURI() + . $reply->find('h3 a', 0)->getAttribute('href'); + + $item['title'] = $reply->find('h3 a', 0)->plaintext; + + $author_html = $reply->find('.author', 0); + // author_html contains the timestamp as text directly inside it, + // so delete all other child elements + foreach($author_html->children as $child) + $child->outertext = ''; + // Timestamps are always in UTC+1 + $item['timestamp'] = trim($author_html->innertext) . ' +01:00'; + + $item['author'] = $reply + ->find('.username, .username-coloured', 0) + ->plaintext; + + $item['content'] = defaultLinkTo($reply->find('.content', 0)->innertext, + $this->getURI()); + + $item['enclosures'] = array(); + foreach($reply->find('.attachbox img.postimage') as $img) + $item['enclosures'][] = urljoin($this->getURI(), $img->src); + + $this->items[] = $item; + } + } +} diff --git a/bridges/QPlayBridge.php b/bridges/QPlayBridge.php new file mode 100644 index 0000000..f204326 --- /dev/null +++ b/bridges/QPlayBridge.php @@ -0,0 +1,132 @@ + array( + 'program' => array( + 'name' => 'Program Name', + 'type' => 'text', + 'required' => true, + ), + ), + 'Catalog' => array( + 'all_pages' => array( + 'name' => 'All Pages', + 'type' => 'checkbox', + 'defaultValue' => false, + ), + ), + ); + + public function getIcon() { + # This should be the favicon served on `self::URI` + return 'https://s3.amazonaws.com/unode1/assets/4957/r3T9Lm9LTLmpAEX6FlSA_apple-touch-icon.png'; + } + + public function getURI() { + switch ($this->queriedContext) { + case 'Program': + return self::URI . '/programs/' . $this->getInput('program'); + case 'Catalog': + return self::URI . '/catalog'; + } + return parent::getURI(); + } + + public function getName() { + switch ($this->queriedContext) { + case 'Program': + $html = getSimpleHTMLDOMCached($this->getURI()) + or returnServerError('Could not load content'); + + return $html->find('h1.program--title', 0)->innertext; + case 'Catalog': + return self::NAME . ' | Programas'; + } + + return parent::getName(); + } + + /* This uses the uscreen platform, other sites can adapt this. https://www.uscreen.tv/ */ + public function collectData() { + switch ($this->queriedContext) { + case 'Program': + $program = $this->getInput('program'); + $html = getSimpleHTMLDOMCached($this->getURI()) + or returnServerError('Could not load content'); + + foreach($html->find('.cce--thumbnails-video-chapter') as $element) { + $cid = $element->getAttribute('data-id'); + $item['title'] = $element->find('.cce--chapter-title', 0)->innertext; + $item['content'] = $element->find('.cce--thumbnails-image-block', 0) + . $element->find('.cce--chapter-body', 0)->innertext; + $item['uri'] = $this->getURI() . '?cid=' . $cid; + + /* TODO: Suport login credentials? */ + /* # Get direct video URL */ + /* $json_source = getContents(self::URI . '/chapters/' . $cid, array('Cookie: _uscreen2_session=???;')) */ + /* or returnServerError('Could not request chapter JSON'); */ + /* $json = json_decode($json_source); */ + + /* $item['enclosures'] = [$json->fallback]; */ + + $this->items[] = $item; + } + + break; + case 'Catalog': + $json_raw = getContents($this->getCatalogURI(1)) + or returnServerError('Could not load catalog content'); + + $json = json_decode($json_raw); + $total_pages = $json->total_pages; + + foreach($this->parseCatalogPage($json) as $item) { + $this->items[] = $item; + } + + if ($this->getInput('all_pages') === true) { + foreach(range(2, $total_pages) as $page) { + $json_raw = getContents($this->getCatalogURI($page)) + or returnServerError('Could not load catalog content (all pages)'); + + $json = json_decode($json_raw); + + foreach($this->parseCatalogPage($json) as $item) { + $this->items[] = $item; + } + } + } + + break; + } + } + + private function getCatalogURI($page) { + return self::URI . '/catalog.json?page=' . $page; + } + + private function parseCatalogPage($json) { + $items = array(); + + foreach($json->records as $record) { + $item = array(); + + $item['title'] = $record->title; + $item['content'] = $record->description + . '
Duration: ' . $record->duration . '
'; + $item['timestamp'] = strtotime($record->release_date); + $item['uri'] = self::URI . $record->url; + $item['enclosures'] = array( + $record->main_poster, + ); + + $items[] = $item; + } + + return $items; + } +} diff --git a/bridges/RadioMelodieBridge.php b/bridges/RadioMelodieBridge.php index ca033fd..fb5aca6 100644 --- a/bridges/RadioMelodieBridge.php +++ b/bridges/RadioMelodieBridge.php @@ -1,34 +1,87 @@ find('div[class=actuitem]'); + $list = $html->find('div[class=displayList]', 0)->children(); foreach($list as $element) { - $item = array(); - - // Get picture URL - $pictureHTML = $element->find('div[class=picture]'); - preg_match( - '/background-image:url\((.*)\);/', - $pictureHTML[0]->getAttribute('style'), - $pictures); - $pictureURL = $pictures[1]; - - $item['enclosures'] = array($pictureURL); - $item['uri'] = self::URI . $element->parent()->href; - $item['title'] = $element->find('h3', 0)->plaintext; - $item['content'] = $element->find('p', 0)->plaintext . '
'; - $this->items[] = $item; + if($element->tag == 'a') { + $articleURL = self::URI . $element->href; + $article = getSimpleHTMLDOM($articleURL); + $textDOM = $article->find('article', 0); + + // Initialise arrays + $item = array(); + $audio = array(); + $picture = array(); + + // Get the Main picture URL + $picture[] = $this->rewriteImage($article->find('div[id=pictureTitleSupport]', 0)->find('img', 0)->src); + $audioHTML = $article->find('audio'); + + // Add the audio element to the enclosure + foreach($audioHTML as $audioElement) { + $audioURL = $audioElement->src; + $audio[] = $audioURL; + } + + // Rewrite pictures URL + $imgs = $textDOM->find('img[src^="http://www.radiomelodie.com/image.php]'); + foreach($imgs as $img) { + $img->src = $this->rewriteImage($img->src); + $article->save(); + } + + // Remove Google Ads + $ads = $article->find('div[class=adInline]'); + foreach($ads as $ad) { + $ad->outertext = ''; + $article->save(); + } + + // Remove Radio Melodie Logo + $logoHTML = $article->find('div[id=logoArticleRM]', 0); + $logoHTML->outertext = ''; + $article->save(); + + $author = $article->find('p[class=AuthorName]', 0)->plaintext; + + $item['enclosures'] = array_merge($picture, $audio); + $item['author'] = $author; + $item['uri'] = $articleURL; + $item['title'] = $article->find('meta[property=og:title]', 0)->content; + $date = $article->find('p[class*=date]', 0)->plaintext; + + // Header Image + $header = ''; + + // Remove the Date and Author part + $textDOM->find('div[class=AuthorDate]', 0)->outertext = ''; + $article->save(); + $text = $textDOM->innertext; + $item['content'] = '

' . $item['title'] . '

' . $date . '
' . $header . $text; + $this->items[] = $item; + } } } + + /* + * Function to rewrite image URL to use the real Image URL and not the resized one (which is very slow) + */ + private function rewriteImage($url) + { + $parts = explode('?', $url); + parse_str(html_entity_decode($parts[1]), $params); + return self::URI . '/' . $params['image']; + + } } diff --git a/bridges/RoadAndTrackBridge.php b/bridges/RoadAndTrackBridge.php new file mode 100644 index 0000000..b3f0acc --- /dev/null +++ b/bridges/RoadAndTrackBridge.php @@ -0,0 +1,68 @@ +find('.custom-promo-title', 0)->href; + $this->items[] = $this->fetchArticle($firstArticleLink); + + $limit = 19; + foreach($page->find('.full-item-title') as $article) { + $this->items[] = $this->fetchArticle($article->href); + $limit -= 1; + if($limit == 0) break; + } + + } + + private function fixImages($content) { + + $enclosures = []; + foreach($content->find('img') as $image) { + $image->src = explode('?', $image->getAttribute('data-src'))[0]; + $enclosures[] = $image->src; + } + + foreach($content->find('.embed-image-wrap, .content-lede-image-wrap') as $imgContainer) { + $imgContainer->style = ''; + } + + return $enclosures; + + } + + private function fetchArticle($articleLink) { + + $articleLink = self::URI . $articleLink; + $article = getSimpleHTMLDOM($articleLink); + $item = array(); + + $item['title'] = $article->find('.content-hed', 0)->innertext; + $item['author'] = $article->find('.byline-name', 0)->innertext; + $item['timestamp'] = strtotime($article->find('.content-info-date', 0)->getAttribute('datetime')); + + $content = $article->find('.content-container', 0); + if($content->find('.content-rail', 0) !== null) + $content->find('.content-rail', 0)->innertext = ''; + $enclosures = $this->fixImages($content); + + $item['enclosures'] = $enclosures; + $item['content'] = $content; + return $item; + + } + + private function getArticleContent($article) { + + return getContents($article->contentUrl); + + } +} diff --git a/bridges/Rue89Bridge.php b/bridges/Rue89Bridge.php index 934ef99..bbb1466 100644 --- a/bridges/Rue89Bridge.php +++ b/bridges/Rue89Bridge.php @@ -9,7 +9,7 @@ class Rue89Bridge extends BridgeAbstract { public function collectData() { $jsonArticles = getContents('https://appdata.nouvelobs.com/rue89/feed.json') - or die('Unable to query Rue89 !'); + or returnServerError('Unable to query Rue89 !'); $articles = json_decode($jsonArticles)->items; foreach($articles as $article) { $this->items[] = $this->getArticle($article); @@ -19,7 +19,8 @@ class Rue89Bridge extends BridgeAbstract { private function getArticle($articleInfo) { - $articleJson = getContents($articleInfo->json_url) or die('Unable to get article !'); + $articleJson = getContents($articleInfo->json_url) + or returnServerError('Unable to get article !'); $article = json_decode($articleJson); $item = array(); $item['title'] = $article->title; diff --git a/bridges/Rule34pahealBridge.php b/bridges/Rule34pahealBridge.php index 1a74616..d130d36 100644 --- a/bridges/Rule34pahealBridge.php +++ b/bridges/Rule34pahealBridge.php @@ -7,4 +7,21 @@ class Rule34pahealBridge extends Shimmie2Bridge { const NAME = 'Rule34paheal'; const URI = 'http://rule34.paheal.net/'; const DESCRIPTION = 'Returns images from given page'; + + protected function getItemFromElement($element){ + $item = array(); + $item['uri'] = $this->getURI() . $element->href; + $item['id'] = (int)preg_replace('/[^0-9]/', '', $element->getAttribute(static::IDATTRIBUTE)); + $item['timestamp'] = time(); + $thumbnailUri = $element->find('img', 0)->src; + $item['tags'] = $element->getAttribute('data-tags'); + $item['title'] = $this->getName() . ' | ' . $item['id']; + $item['content'] = '
Tags: ' + . $item['tags']; + return $item; + } } diff --git a/bridges/SIMARBridge.php b/bridges/SIMARBridge.php new file mode 100644 index 0000000..1e446cf --- /dev/null +++ b/bridges/SIMARBridge.php @@ -0,0 +1,63 @@ + array( + 'interventions' => array( + 'type' => 'checkbox', + 'name' => 'Incluir Intervenções?', + 'defaultValue' => 'checked', + ) + ) + ); + + public function collectData() { + $html = getSimpleHTMLDOM(self::getURI()) + or returnServerError('Could not load content'); + $e_home = $html->find('#home', 0) + or returnServerError('Invalid site structure'); + + foreach($e_home->find('span') as $element) { + $item = array(); + + $item['title'] = 'Rotura: ' . $element->plaintext; + $item['content'] = $element->innertext; + $item['uid'] = 'urn:sha1:' . hash('sha1', $item['content']); + + $this->items[] = $item; + } + + if ($this->getInput('interventions')) { + $e_main1 = $html->find('#menu1', 0) + or returnServerError('Invalid site structure'); + + foreach ($e_main1->find('a') as $element) { + $item = array(); + + $item['title'] = 'Intervenção: ' . $element->plaintext; + $item['uri'] = self::getURI() . $element->href; + $item['content'] = $element->innertext; + + /* Try to get the actual contents for this kind of item */ + $item_html = getSimpleHTMLDOMCached($item['uri']); + if ($item_html) { + $e_item = $item_html->find('.auto-style59', 0); + foreach($e_item->find('p') as $paragraph) { + /* Remove empty paragraphs */ + if (preg_match('/^(\W| )+$/', $paragraph->innertext) == 1) { + $paragraph->outertext = ''; + } + } + if ($e_item) { + $item['content'] = $e_item->innertext; + } + } + + $this->items[] = $item; + } + } + } +} diff --git a/bridges/SakugabooruBridge.php b/bridges/SakugabooruBridge.php deleted file mode 100644 index 1d6cee0..0000000 --- a/bridges/SakugabooruBridge.php +++ /dev/null @@ -1,11 +0,0 @@ - array( + 'name' => 'Minimum Episodes', + 'type' => 'number', + 'title' => 'Minimum number of episodes before including in feed', + 'defaultValue' => 0, + ), + 'min_total_episodes' => array( + 'name' => 'Minimum Total Episodes', + 'type' => 'number', + 'title' => 'Minimum total number of episodes before including in feed', + 'defaultValue' => 0, + ), + 'require_banner' => array( + 'name' => 'Require Banner', + 'type' => 'checkbox', + 'title' => 'Only include anime with custom banner image', + 'defaultValue' => false, + ), + ), + ); + + private $uri; + + public function getURI() { + return isset($this->uri) ? $this->uri : parent::getURI(); + } + + public function collectData(){ + $html = $this->loadSeasonAnimeList(); + + $animes = $html->find('div.header_display_box_info') + or returnServerError('Could not find anime headers!'); + + $min_episodes = $this->getInput('min_episodes') ?: 0; + $min_total_episodes = $this->getInput('min_total_episodes') ?: 0; + + foreach($animes as $anime) { + + list( + $episodes_released, + /* of */, + $episodes_total + ) = explode(' ', $this->extractAnimeEpisodeInformation($anime)); + + // Skip if not enough episodes yet + if ($episodes_released < $min_episodes) { + continue; + } + + // Skip if too many episodes in total + if ($episodes_total !== '?' && $episodes_total < $min_total_episodes) { + continue; + } + + // Skip if https://static.shanaproject.com/no-art.jpg + if ($this->getInput('require_banner') + && strpos($this->extractAnimeBackgroundImage($anime), 'no-art') !== false) { + continue; + } + + $this->items[] = array( + 'title' => $this->extractAnimeTitle($anime), + 'author' => $this->extractAnimeAuthor($anime), + 'uri' => $this->extractAnimeUri($anime), + 'timestamp' => $this->extractAnimeTimestamp($anime), + 'content' => $this->buildAnimeContent($anime), + ); + + } + } // Returns an html object for the Season Anime List (latest season) private function loadSeasonAnimeList(){ - // First we need to find the URI to the latest season from the - // 'seasons' page searching for 'Season Anime List' - $html = getSimpleHTMLDOM($this->getURI() . '/seasons'); - if(!$html) - returnServerError('Could not load \'seasons\' page!'); - - $season = $html->find('div.follows_menu/a', 1); - if(!$season) - returnServerError('Could not find \'Season Anime List\'!'); - - $html = getSimpleHTMLDOM($this->getURI() . $season->href); - if(!$html) - returnServerError( + + $html = getSimpleHTMLDOM(self::URI . '/seasons') + or returnServerError('Could not load \'seasons\' page!'); + + $html = defaultLinkTo($html, self::URI . '/seasons'); + + $season = $html->find('div.follows_menu > a', 1) + or returnServerError('Could not find \'Season Anime List\'!'); + + $html = getSimpleHTMLDOM($season->href) + or returnServerError( 'Could not load \'Season Anime List\' from \'' . $season->innertext . '\'!' ); + $this->uri = $season->href; + + $html = defaultLinkTo($html, $season->href); + return $html; + } // Extracts the anime title private function extractAnimeTitle($anime){ - $title = $anime->find('a', 0); - if(!$title) - returnServerError('Could not find anime title!'); + $title = $anime->find('a', 0) + or returnServerError('Could not find anime title!'); return trim($title->innertext); } // Extracts the anime URI private function extractAnimeUri($anime){ - $uri = $anime->find('a', 0); - if(!$uri) - returnServerError('Could not find anime URI!'); - return $this->getURI() . $uri->href; + $uri = $anime->find('a', 0) + or returnServerError('Could not find anime URI!'); + return $uri->href; } // Extracts the anime release date (timestamp) private function extractAnimeTimestamp($anime){ $timestamp = $anime->find('span.header_info_block', 1); - if(!$timestamp) + + if(!$timestamp) { return null; + } + return strtotime($timestamp->innertext); } // Extracts the anime studio name (author) private function extractAnimeAuthor($anime){ $author = $anime->find('span.header_info_block', 2); - if(!$author) - return; // Sometimes the studio is unknown, so leave empty + + if(!$author) { + return null; // Sometimes the studio is unknown, so leave empty + } + return trim($author->innertext); } // Extracts the episode information (x of y released) private function extractAnimeEpisodeInformation($anime){ - $episode = $anime->find('div.header_info_episode', 0); - if(!$episode) - returnServerError('Could not find anime episode information!'); - return preg_replace('/\r|\n/', ' ', $episode->plaintext); + $episode = $anime->find('div.header_info_episode', 0) + or returnServerError('Could not find anime episode information!'); + + $retVal = preg_replace('/\r|\n/', ' ', $episode->plaintext); + $retVal = preg_replace('/\s+/', ' ', $retVal); + + return $retVal; } // Extracts the background image @@ -73,15 +155,16 @@ class ShanaprojectBridge extends BridgeAbstract { // Getting the picture is a little bit tricky as it is part of the style. // Luckily the style is part of the parent div :) - if(preg_match('/url\(\/\/([^\)]+)\)/i', $anime->parent->style, $matches)) + if(preg_match('/url\(\/\/([^\)]+)\)/i', $anime->parent->style, $matches)) { return $matches[1]; + } returnServerError('Could not extract background image!'); } // Builds an URI to search for a specific anime (subber is left empty) private function buildAnimeSearchUri($anime){ - return $this->getURI() + return self::URI . '/search/?title=' . urlencode($this->extractAnimeTitle($anime)) . '&subber='; @@ -102,22 +185,4 @@ class ShanaprojectBridge extends BridgeAbstract { . $this->buildAnimeSearchUri($anime) . '">Search episodes

'; } - - public function collectData(){ - $html = $this->loadSeasonAnimeList(); - - $animes = $html->find('div.header_display_box_info'); - if(!$animes) - returnServerError('Could not find anime headers!'); - - foreach($animes as $anime) { - $item = array(); - $item['title'] = $this->extractAnimeTitle($anime); - $item['author'] = $this->extractAnimeAuthor($anime); - $item['uri'] = $this->extractAnimeUri($anime); - $item['timestamp'] = $this->extractAnimeTimestamp($anime); - $item['content'] = $this->buildAnimeContent($anime); - $this->items[] = $item; - } - } } diff --git a/bridges/SkimfeedBridge.php b/bridges/SkimfeedBridge.php index 9fdd454..1b78baf 100644 --- a/bridges/SkimfeedBridge.php +++ b/bridges/SkimfeedBridge.php @@ -18,7 +18,6 @@ class SkimfeedBridge extends BridgeAbstract { 'box_channel' => array( 'name' => 'Channel', 'type' => 'list', - 'required' => true, 'title' => 'Select your channel', 'values' => array( 'Hacker News' => '/news/hacker-news.html', @@ -68,7 +67,6 @@ class SkimfeedBridge extends BridgeAbstract { 'tech_channel' => array( 'name' => 'Tech channel', 'type' => 'list', - 'required' => true, 'title' => 'Select your tech channel', 'values' => array( 'Agg' => array( diff --git a/bridges/SoundcloudBridge.php b/bridges/SoundcloudBridge.php index 91ac2b5..8938ff9 100644 --- a/bridges/SoundcloudBridge.php +++ b/bridges/SoundcloudBridge.php @@ -14,7 +14,9 @@ class SoundCloudBridge extends BridgeAbstract { ) )); - const CLIENT_ID = '4jkoEFmZEDaqjwJ9Eih4ATNhcH3vMVfp'; + const CLIENT_ID = 'W0KEWWILAjDiRH89X0jpwzuq6rbSK08R'; + + private $feedIcon = null; public function collectData(){ @@ -25,6 +27,8 @@ class SoundCloudBridge extends BridgeAbstract { . self::CLIENT_ID )) or returnServerError('No results for this query'); + $this->feedIcon = $res->avatar_url; + $tracks = json_decode(getContents( 'https://api.soundcloud.com/users/' . urlencode($res->id) @@ -56,6 +60,14 @@ class SoundCloudBridge extends BridgeAbstract { } + public function getIcon(){ + if ($this->feedIcon) { + return $this->feedIcon; + } + + return parent::getIcon(); + } + public function getName(){ if(!is_null($this->getInput('u'))) { return self::NAME . ' - ' . $this->getInput('u'); diff --git a/bridges/SplCenterBridge.php b/bridges/SplCenterBridge.php new file mode 100644 index 0000000..7a69090 --- /dev/null +++ b/bridges/SplCenterBridge.php @@ -0,0 +1,64 @@ + array( + 'name' => 'Content', + 'type' => 'list', + 'values' => array( + 'News' => 'news', + 'Hatewatch' => 'hatewatch', + ), + 'defaultValue' => 'news', + ) + ) + ); + + const CACHE_TIMEOUT = 3600; // 1 hour + + protected function parseItem($item) { + $item = parent::parseItem($item); + + $articleHtml = getSimpleHTMLDOMCached($item['uri']) + or returnServerError('Could not request: ' . $item['uri']); + + foreach ($articleHtml->find('.file') as $index => $media) { + $articleHtml->find('div.file', $index)->outertext = '' . $media->outertext . ''; + } + + $item['content'] = $articleHtml->find('div#group-content-container', 0)->innertext; + $item['enclosures'][] = $articleHtml->find('meta[name="twitter:image"]', 0)->content; + + return $item; + } + + public function collectData() { + $this->collectExpandableDatas($this->getURI() . '/rss.xml'); + } + + public function getURI() { + + if (!is_null($this->getInput('content'))) { + return self::URI . '/' . $this->getInput('content'); + } + + return parent::getURI(); + } + + public function getName() { + + if (!is_null($this->getInput('content'))) { + $parameters = $this->getParameters(); + + $contentValues = array_flip($parameters[0]['content']['values']); + + return $contentValues[$this->getInput('content')] . ' - Southern Poverty Law Center'; + } + + return parent::getName(); + } +} diff --git a/bridges/SteamBridge.php b/bridges/SteamBridge.php index 8ff456d..d0acd6d 100644 --- a/bridges/SteamBridge.php +++ b/bridges/SteamBridge.php @@ -8,44 +8,12 @@ class SteamBridge extends BridgeAbstract { const MAINTAINER = 'jacknumber'; const PARAMETERS = array( 'Wishlist' => array( - 'username' => array( - 'name' => 'Username', + 'userid' => array( + 'name' => 'Steamid64 (find it on steamid.io)', + 'title' => 'User ID (17 digits). Find your user ID with steamid.io or steamidfinder.com', 'required' => true, - ), - 'currency' => array( - 'name' => 'Currency', - 'type' => 'list', - 'values' => array( - // source: http://steam.steamlytics.xyz/currencies - 'USD' => 'us', - 'GBP' => 'gb', - 'EUR' => 'fr', - 'CHF' => 'ch', - 'RUB' => 'ru', - 'BRL' => 'br', - 'JPY' => 'jp', - 'SEK' => 'se', - 'IDR' => 'id', - 'MYR' => 'my', - 'PHP' => 'ph', - 'SGD' => 'sg', - 'THB' => 'th', - 'KRW' => 'kr', - 'TRY' => 'tr', - 'MXN' => 'mx', - 'CAD' => 'ca', - 'NZD' => 'nz', - 'CNY' => 'cn', - 'INR' => 'in', - 'CLP' => 'cl', - 'PEN' => 'pe', - 'COP' => 'co', - 'ZAR' => 'za', - 'HKD' => 'hk', - 'TWD' => 'tw', - 'SRD' => 'sr', - 'AED' => 'ae', - ), + 'exampleValue' => '76561198821231205', + 'pattern' => '[0-9]{17}', ), 'only_discount' => array( 'name' => 'Only discount', @@ -56,27 +24,15 @@ class SteamBridge extends BridgeAbstract { public function collectData(){ - $username = $this->getInput('username'); - $params = array( - 'cc' => $this->getInput('currency') - ); - - $url = self::URI . 'wishlist/id/' . $username . '?' . http_build_query($params); + $userid = $this->getInput('userid'); - $targetVariable = 'g_rgAppInfo'; + $sourceUrl = self::URI . 'wishlist/profiles/' . $userid . '/wishlistdata?p=0'; $sort = array(); - $html = ''; - $html = getSimpleHTMLDOM($url) - or returnServerError("Could not request Steam Wishlist. Tried:\n - $url"); + $json = getContents($sourceUrl) + or returnServerError('Could not get content from wishlistdata (' . $sourceUrl . ')'); - $jsContent = $html->find('.responsive_page_template_content script', 0)->innertext; - - if(preg_match('/var ' . $targetVariable . ' = (.*?);/s', $jsContent, $matches)) { - $appsData = json_decode($matches[1]); - } else { - returnServerError("Could not parse JS variable ($targetVariable) in page content."); - } + $appsData = json_decode($json); foreach($appsData as $id => $element) { @@ -87,6 +43,8 @@ class SteamBridge extends BridgeAbstract { if($element->subs) { $appIsBuyable = 1; + $priceBlock = str_get_html($element->subs[0]->discount_block); + $appPrice = str_replace('--', '00', $priceBlock->find('.discount_final_price', 0)->plaintext); if($element->subs[0]->discount_pct) { @@ -94,8 +52,6 @@ class SteamBridge extends BridgeAbstract { $discountBlock = str_get_html($element->subs[0]->discount_block); $appDiscountValue = $discountBlock->find('.discount_pct', 0)->plaintext; $appOldPrice = $discountBlock->find('.discount_original_price', 0)->plaintext; - $appNewPrice = $discountBlock->find('.discount_final_price', 0)->plaintext; - $appPrice = $appNewPrice; } else { @@ -103,7 +59,6 @@ class SteamBridge extends BridgeAbstract { continue; } - $appPrice = $element->subs[0]->price / 100; } } else { @@ -117,11 +72,14 @@ class SteamBridge extends BridgeAbstract { } } + $coverUrl = str_replace('_292x136', '', strtok($element->capsule, '?')); + $picturesPath = pathinfo($coverUrl)['dirname'] . '/'; + $item = array(); $item['uri'] = "http://store.steampowered.com/app/$id/"; $item['title'] = $element->name; $item['type'] = $appType; - $item['cover'] = str_replace('_292x136', '', $element->capsule); + $item['cover'] = $coverUrl; $item['timestamp'] = $element->added; $item['isBuyable'] = $appIsBuyable; $item['hasDiscount'] = $appHasDiscount; @@ -129,22 +87,29 @@ class SteamBridge extends BridgeAbstract { $item['priority'] = $element->priority; if($appIsBuyable) { + $item['price'] = floatval(str_replace(',', '.', $appPrice)); + $item['content'] = $appPrice; + + } + + if($appIsFree) { + $item['content'] = 'Free'; } if($appHasDiscount) { $item['discount']['value'] = $appDiscountValue; - $item['discount']['oldPrice'] = floatval(str_replace(',', '.', $appOldPrice)); - $item['discount']['newPrice'] = floatval(str_replace(',', '.', $appNewPrice)); + $item['discount']['oldPrice'] = $appOldPrice; + $item['content'] = '' . $appOldPrice . ' ' . $appPrice . ' (' . $appDiscountValue . ')'; } $item['enclosures'] = array(); - $item['enclosures'][] = str_replace('_292x136', '', $element->capsule); + $item['enclosures'][] = $coverUrl; - foreach($element->screenshots as $screenshot) { - $item['enclosures'][] = substr($element->capsule, 0, -31) . $screenshot; + foreach($element->screenshots as $screenshotFileName) { + $item['enclosures'][] = $picturesPath . $screenshotFileName; } $sort[$id] = $element->priority; diff --git a/bridges/SteamCommunityBridge.php b/bridges/SteamCommunityBridge.php new file mode 100644 index 0000000..9919a4b --- /dev/null +++ b/bridges/SteamCommunityBridge.php @@ -0,0 +1,191 @@ + array( + 'name' => 'App ID', + 'required' => true + ), + 'category' => array( + 'name' => 'category', + 'type' => 'list', + 'exampleValue' => 'Artwork', + 'title' => 'Select a category', + 'values' => array( + 'Artwork' => 'images', + 'Screenshots' => 'screenshots', + 'Videos' => 'videos', + 'Workshop' => 'workshop' + ) + ) + ) + ); + + public function getIcon() { + return self::URI . '/favicon.ico'; + } + + protected function getMainPage() { + $category = $this->getInput('category'); + $html = getSimpleHTMLDOM($this->getURI()) + or returnServerError('Could not fetch Steam data.'); + + return $html; + } + + public function getName() { + $category = $this->getInput('category'); + + if (is_null('i') || is_null($category)) { + return self::NAME; + } + + $html = $this->getMainPage(); + + $titleItem = $html->find('div.apphub_AppName', 0); + + if (!$titleItem) + return self::NAME; + + return $titleItem->innertext . ' (' . ucwords($category) . ')'; + } + + public function getURI() { + if ($this->getInput('category') === 'workshop') + return self::URI . '/workshop/browse/?appid=' + . $this->getInput('i') . '&browsesort=mostrecent'; + + return self::URI . '/app/' + . $this->getInput('i') . '/' + . $this->getInput('category') + . '/?p=1&browsefilter=mostrecent'; + } + + private function collectMedia() { + $category = $this->getInput('category'); + $html = $this->getMainPage(); + $cards = $html->find('div.apphub_Card'); + + foreach($cards as $card) { + $uri = $card->getAttribute('data-modal-content-url'); + + $htmlCard = getSimpleHTMLDOMCached($uri); + + $author = $card->find('div.apphub_CardContentAuthorName', 0)->innertext; + $author = strip_tags($author); + + $title = $author . '\'s screenshot'; + + if ($category != 'screenshots') + $title = $htmlCard->find('div.workshopItemTitle', 0)->innertext; + + $date = $htmlCard->find('div.detailsStatRight', 0)->innertext; + + // create item + $item = array(); + $item['title'] = $title; + $item['uri'] = $uri; + $item['timestamp'] = strtotime($date); + $item['author'] = $author; + $item['categories'] = $category; + + $media = $htmlCard->getElementById('ActualMedia'); + $mediaURI = $media->getAttribute('src'); + $downloadURI = $mediaURI; + + if ($category == 'videos') { + preg_match('/.*\/embed\/(.*)\?/', $mediaURI, $result); + $youtubeID = $result[1]; + $mediaURI = 'https://img.youtube.com/vi/' . $youtubeID . '/hqdefault.jpg'; + $downloadURI = 'https://www.youtube.com/watch?v=' . $youtubeID; + } + + $desc = ''; + + if ($category == 'screenshots') { + $descItem = $htmlCard->find('div.screenshotDescription', 0); + if ($descItem) + $desc = $descItem->innertext; + } + + if ($category == 'images') { + $descItem = $htmlCard->find('div.nonScreenshotDescription', 0); + if ($descItem) + $desc = $descItem->innertext; + $downloadURI = $htmlCard->find('a.downloadImage', 0)->href; + } + + $item['content'] = '

'; + $item['content'] .= '

' . $desc . '

'; + + $this->items[] = $item; + + if (count($this->items) >= 10) + break; + } + } + + private function collectWorkshop() { + $category = $this->getInput('category'); + $html = $this->getMainPage(); + $workShopItems = $html->find('div.workshopItem'); + + foreach($workShopItems as $workShopItem) { + $author = $workShopItem->find('div.workshopItemAuthorName', 0)->find('a', 0); + $author = $author->innertext; + + $fileRating = $workShopItem->find('img.fileRating', 0); + + $uri = $workShopItem->find('a.ugc', 0)->getAttribute('href'); + + $htmlItem = getSimpleHTMLDOMCached($uri); + + $title = $htmlItem->find('div.workshopItemTitle', 0)->innertext; + $date = $htmlItem->find('div.detailsStatRight', 0)->innertext; + $description = $htmlItem->find('div.workshopItemDescription', 0)->innertext; + + $previewImage = $htmlItem->find('#previewImage', 0); + + $htmlTags = $htmlItem->find('div.workshopTags'); + + $tags = ''; + + foreach($htmlTags as $htmlTag) { + if ($tags !== '') + $tags .= ','; + + $tags .= $htmlTag->find('a', 0)->innertext; + } + + // create item + $item = array(); + $item['title'] = $title; + $item['uri'] = $uri; + $item['timestamp'] = strtotime($date); + $item['author'] = $author; + $item['categories'] = $category; + + $item['content'] = '

' + . $previewImage . '

' . $fileRating + . '

' . $description . '

'; + + $this->items[] = $item; + + if (count($this->items) >= 10) + break; + } + } + + public function collectData() { + if ($this->getInput('category') === 'workshop') + $this->collectWorkshop(); + else + $this->collectMedia(); + } +} diff --git a/bridges/StockFilingsBridge.php b/bridges/StockFilingsBridge.php new file mode 100644 index 0000000..f774244 --- /dev/null +++ b/bridges/StockFilingsBridge.php @@ -0,0 +1,80 @@ + array( + 'name' => 'cik', + 'required' => true, + 'exampleValue' => 'AMD', + // https://stackoverflow.com/a/12827734 + 'pattern' => '[A-Za-z0-9]+', + ), + )); + + public function getIcon() { + return 'https://www.sec.gov/favicon.ico'; + } + + /** + * Generates search URL + */ + private function getSearchUrl() { + return self::SEARCH_URL . $this->getInput('ticker'); + } + + /** + * Returns the Company Name + */ + private function getRssFeed($html) { + $links = $html->find('#contentDiv a'); + + foreach ($links as $link) { + $href = $link->href; + + if (substr($href, 0, 4) !== 'http') { + $href = self::WEBSITE_ROOT . $href; + } + parse_str(html_entity_decode(parse_url($href, PHP_URL_QUERY)), $query); + + if (isset($query['output']) and ($query['output'] == 'atom')) { + return $href; + } + } + + return false; + } + + /** + * Return \simple_html_dom object + * for the entire html of the product page + */ + private function getHtml() { + $uri = $this->getSearchUrl(); + + return getSimpleHTMLDOM($uri) ?: returnServerError('Could not request SEC.'); + } + + /** + * Scrape the SEC Stock Filings RSS Feed URL + * and redirect there + */ + public function collectData() { + $html = $this->getHtml(); + $rssFeedUrl = $this->getRssFeed($html); + + if ($rssFeedUrl) { + parent::collectExpandableDatas($rssFeedUrl); + } else { + returnClientError('Could not find RSS Feed URL. Are you sure you used a valid CIK?'); + } + } +} diff --git a/bridges/StoriesIGBridge.php b/bridges/StoriesIGBridge.php new file mode 100644 index 0000000..ddf9846 --- /dev/null +++ b/bridges/StoriesIGBridge.php @@ -0,0 +1,47 @@ + array( + 'name' => 'Instagram username', + 'type' => 'text', + 'required' => true, + 'title' => 'Insert the username here' + ), + ) + ); + + public function collectData(){ + $html = getSimpleHTMLDOM($this->getURI()) + or returnServerError('Failed to receive ' . $this->getURI()); + + $results = $html->find('article'); + + foreach($results as $result) { + + $item = array(); + + $item['title'] = $this->getInput('username') . ' story'; + $item['uri'] = $result->find('div.download', 0)->find('a', 0)->href; + $item['author'] = $this->getInput('username'); + $item['uid'] = $result->find('time', 0)->datetime; + + $item['content'] = $result; + + $this->items[] = $item; + } + } + + public function getURI(){ + $uri = self::URI . '/stories/'; + $uri .= urlencode($this->getInput('username')); + return $uri; + + return parent::getURI(); + } +} diff --git a/bridges/TelegramBridge.php b/bridges/TelegramBridge.php new file mode 100644 index 0000000..3afc283 --- /dev/null +++ b/bridges/TelegramBridge.php @@ -0,0 +1,301 @@ + array( + 'name' => 'Username', + 'type' => 'text', + 'required' => true, + 'exampleValue' => '@telegram', + ) + ) + ); + + const CACHE_TIMEOUT = 900; // 15 mins + + private $feedName = ''; + private $enclosures = array(); + private $itemTitle = ''; + + private $backgroundImageRegex = "/background-image:url\('(.*)'\)/"; + + public function collectData() { + + $html = getSimpleHTMLDOM($this->getURI()) + or returnServerError('Could not request: ' . $this->getURI()); + + $channelTitle = htmlspecialchars_decode( + $html->find('div.tgme_channel_info_header_title span', 0)->plaintext, + ENT_QUOTES + ); + $this->feedName = $channelTitle . ' (@' . $this->processUsername() . ')'; + + foreach($html->find('div.tgme_widget_message_wrap.js-widget_message_wrap') as $index => $messageDiv) { + $this->itemTitle = ''; + $this->enclosures = array(); + $item = array(); + + $item['uri'] = $this->processUri($messageDiv); + $item['content'] = html_entity_decode($this->processContent($messageDiv), ENT_QUOTES); + $item['title'] = html_entity_decode($this->itemTitle, ENT_QUOTES); + $item['timestamp'] = $this->processDate($messageDiv); + $item['enclosures'] = $this->enclosures; + $author = trim($messageDiv->find('a.tgme_widget_message_owner_name', 0)->plaintext); + $item['author'] = html_entity_decode($author, ENT_QUOTES); + + $this->items[] = $item; + } + $this->items = array_reverse($this->items); + } + + public function getURI() { + + if (!is_null($this->getInput('username'))) { + return self::URI . '/s/' . $this->processUsername(); + } + + return parent::getURI(); + } + + public function getName() { + + if (!empty($this->feedName)) { + return $this->feedName . ' - Telegram'; + } + + return parent::getName(); + } + + private function processUsername() { + + if (substr($this->getInput('username'), 0, 1) === '@') { + return substr($this->getInput('username'), 1); + } + + return $this->getInput('username'); + } + + private function processUri($messageDiv) { + return $messageDiv->find('a.tgme_widget_message_date', 0)->href; + } + + private function processContent($messageDiv) { + $message = ''; + + if ($messageDiv->find('div.tgme_widget_message_forwarded_from', 0)) { + $message = $messageDiv->find('div.tgme_widget_message_forwarded_from', 0)->innertext . '

'; + } + + if ($messageDiv->find('a.tgme_widget_message_reply', 0)) { + $message = $this->processReply($messageDiv); + } + + if ($messageDiv->find('div.tgme_widget_message_sticker_wrap', 0)) { + $message .= $this->processSticker($messageDiv); + } + + if ($messageDiv->find('div.tgme_widget_message_poll', 0)) { + $message .= $this->processPoll($messageDiv); + } + + if ($messageDiv->find('video', 0)) { + $message .= $this->processVideo($messageDiv); + } + + if ($messageDiv->find('a.tgme_widget_message_photo_wrap', 0)) { + $message .= $this->processPhoto($messageDiv); + } + + if ($messageDiv->find('a.not_supported', 0)) { + $message .= $this->processNotSupported($messageDiv); + } + + if ($messageDiv->find('div.tgme_widget_message_text.js-message_text', 0)) { + $message .= $messageDiv->find('div.tgme_widget_message_text.js-message_text', 0); + + $this->itemTitle = $this->ellipsisTitle( + $messageDiv->find('div.tgme_widget_message_text.js-message_text', 0)->plaintext + ); + } + + if ($messageDiv->find('a.tgme_widget_message_link_preview', 0)) { + $message .= $this->processLinkPreview($messageDiv); + } + + return $message; + } + + private function processReply($messageDiv) { + + $reply = $messageDiv->find('a.tgme_widget_message_reply', 0); + + return <<{$reply->find('span.tgme_widget_message_author_name', 0)->plaintext}
+{$reply->find('div.tgme_widget_message_text', 0)->innertext} +{$reply->href}
+EOD; + } + + private function processSticker($messageDiv) { + + if (empty($this->itemTitle)) { + $this->itemTitle = '@' . $this->processUsername() . ' posted a sticker'; + } + + $stickerDiv = $messageDiv->find('div.tgme_widget_message_sticker_wrap', 0); + + preg_match($this->backgroundImageRegex, $stickerDiv->find('i', 0)->style, $sticker); + + $this->enclosures[] = $sticker[1]; + + return << +EOD; + } + + private function processPoll($messageDiv) { + + $poll = $messageDiv->find('div.tgme_widget_message_poll', 0); + + $title = $poll->find('div.tgme_widget_message_poll_question', 0)->plaintext; + $type = $poll->find('div.tgme_widget_message_poll_type', 0)->plaintext; + + if (empty($this->itemTitle)) { + $this->itemTitle = $title; + } + + $pollOptions = '
    '; + + foreach ($poll->find('div.tgme_widget_message_poll_option') as $option) { + $pollOptions .= '
  • ' . $option->children(0)->plaintext . ' - ' . + $option->find('div.tgme_widget_message_poll_option_text', 0)->plaintext . '
    • '; + } + $pollOptions .= '
'; + + return <<$type
{$pollOptions} +EOD; + } + + private function processLinkPreview($messageDiv) { + + $image = ''; + $title = ''; + $site = ''; + $description = ''; + + $preview = $messageDiv->find('a.tgme_widget_message_link_preview', 0); + + if (trim($preview->innertext) === '') { + return ''; + } + + if($preview->find('i', 0) && + preg_match($this->backgroundImageRegex, $preview->find('i', 0)->style, $photo)) { + + $image = ''; + $this->enclosures[] = $photo[1]; + } + + if ($preview->find('div.link_preview_title', 0)) { + $title = $preview->find('div.link_preview_title', 0)->plaintext; + } + + if ($preview->find('div.link_preview_site_name', 0)) { + $site = $preview->find('div.link_preview_site_name', 0)->plaintext; + } + + if ($preview->find('div.link_preview_description', 0)) { + $description = $preview->find('div.link_preview_description', 0)->plaintext; + } + + return <<$image
+{$title} - {$site}
{$description} +EOD; + } + + private function processVideo($messageDiv) { + + if (empty($this->itemTitle)) { + $this->itemTitle = '@' . $this->processUsername() . ' posted a video'; + } + + if ($messageDiv->find('i.tgme_widget_message_video_thumb')) { + preg_match($this->backgroundImageRegex, $messageDiv->find('i.tgme_widget_message_video_thumb', 0)->style, $photo); + } elseif ($messageDiv->find('i.link_preview_video_thumb')) { + preg_match($this->backgroundImageRegex, $messageDiv->find('i.link_preview_video_thumb', 0)->style, $photo); + } + + $this->enclosures[] = $photo[1]; + + return << + + +EOD; + } + + private function processPhoto($messageDiv) { + + if (empty($this->itemTitle)) { + $this->itemTitle = '@' . $this->processUsername() . ' posted a photo'; + } + + $photos = ''; + + foreach ($messageDiv->find('a.tgme_widget_message_photo_wrap') as $photoWrap) { + preg_match($this->backgroundImageRegex, $photoWrap->style, $photo); + + $this->enclosures[] = $photo[1]; + + $photos .= <<
+EOD; + } + return $photos; + } + + private function processNotSupported($messageDiv) { + + if (empty($this->itemTitle)) { + $this->itemTitle = '@' . $this->processUsername() . ' posted a video'; + } + + if ($messageDiv->find('i.tgme_widget_message_video_thumb')) { + preg_match($this->backgroundImageRegex, $messageDiv->find('i.tgme_widget_message_video_thumb', 0)->style, $photo); + } elseif ($messageDiv->find('i.link_preview_video_thumb')) { + preg_match($this->backgroundImageRegex, $messageDiv->find('i.link_preview_video_thumb', 0)->style, $photo); + } + + $this->enclosures[] = $photo[1]; + + return << +{$messageDiv->find('div.message_media_not_supported_label', 0)->innertext}

+{$messageDiv->find('span.message_media_view_in_telegram', 0)->innertext}

+ +EOD; + } + + private function processDate($messageDiv) { + + $messageMeta = $messageDiv->find('span.tgme_widget_message_meta', 0); + return $messageMeta->find('time', 0)->datetime; + + } + + private function ellipsisTitle($text) { + + $length = 100; + + if (strlen($text) > $length) { + $text = explode('
', wordwrap($text, $length, '
')); + return $text[0] . '...'; + } + return $text; + } +} diff --git a/bridges/TheGuardianBridge.php b/bridges/TheGuardianBridge.php new file mode 100644 index 0000000..e655f0e --- /dev/null +++ b/bridges/TheGuardianBridge.php @@ -0,0 +1,96 @@ + array( + 'name' => 'Feed', + 'type' => 'list', + 'values' => array( + 'World News' => 'world/rss', + 'US News' => '/us-news/rss', + 'UK News' => '/uk-news/rss', + 'Europe News' => '/world/europe-news/rss', + 'Asia News' => '/world/asia/rss', + 'Tech' => '/uk/technology/rss', + 'Business News' => '/uk/business/rss', + 'Opinion' => '/uk/commentisfree/rss', + 'Lifestyle' => '/uk/lifeandstyle/rss', + 'Culture' => '/uk/culture/rss', + 'Sports' => '/uk/sport/rss' + ) + ) + + /* + + Topicwise Links + + You can find the base feed for any topic by appending /rss to the url. + + Example: + + https://feeds.theguardian.com/theguardian/uk-news/rss + https://feeds.theguardian.com/theguardian/us-news/rss + + Or simply + + https://www.theguardian.com/world/rss + + Just add that topic as a value in the PARAMETERS const. + + */ + + + )); + + public function collectData(){ + $feed = $this->getInput('feed'); + $feedURL = 'https://feeds.theguardian.com/theguardian/' . $feed; + $this->collectExpandableDatas($feedURL, 10); + } + + protected function parseItem($newsItem){ + $item = parent::parseItem($newsItem); + + // --- Recovering the article --- + + // $articlePage gets the entire page's contents + $articlePage = getSimpleHTMLDOM($newsItem->link); + // figure contain's the main article image + $article = $articlePage->find('figure', 0); + // content__article-body has the actual article + foreach($articlePage->find('.content__article-body') as $element) + $article = $article . $element; + + // --- Fixing ugly elements --- + + // Replace the image viewer and BS with the image itself + foreach($articlePage->find('a.article__img-container') as $uslElementLoc) { + $main_img = $uslElementLoc->find('img', 0); + $article = str_replace($uslElementLoc, $main_img, $article); + } + + // List of all the crap in the article + $uselessElements = array( + '#show-caption', + '.element-atom', + '.submeta', + 'youtube-media-atom', + 'svg' + ); + + // Remove the listed crap + foreach($uselessElements as $uslElement) { + foreach($articlePage->find($uslElement) as $uslElementLoc) { + $article = str_replace($uslElementLoc, '', $article); + } + } + + $item['content'] = $article; + + return $item; + } +} diff --git a/bridges/ThePirateBayBridge.php b/bridges/ThePirateBayBridge.php index 9aefcbb..5fc04eb 100644 --- a/bridges/ThePirateBayBridge.php +++ b/bridges/ThePirateBayBridge.php @@ -3,7 +3,7 @@ class ThePirateBayBridge extends BridgeAbstract { const MAINTAINER = 'mitsukarenai'; const NAME = 'The Pirate Bay'; - const URI = 'https://thepiratebay.wf/'; + const URI = 'https://thepiratebay.org/'; const DESCRIPTION = 'Returns results for the keywords. You can put several list of keywords by separating them with a semicolon (e.g. "one show;another show"). Category based search needs the category number as input. User based @@ -149,11 +149,12 @@ class ThePirateBayBridge extends BridgeAbstract { || !is_null($element->find('img[alt=VIP]', 0)) || !is_null($element->find('img[alt=Trusted]', 0))) { $item = array(); - $item['uri'] = $element->find('a', 3)->href; + $item['uri'] = self::URI . $element->find('a.detLink', 0)->href; $item['id'] = self::URI . $element->find('a.detLink', 0)->href; $item['timestamp'] = parseDateTimestamp($element); $item['author'] = $element->find('a.detDesc', 0)->plaintext; $item['title'] = $element->find('a.detLink', 0)->plaintext; + $item['magnet'] = $element->find('a', 3)->href; $item['seeders'] = (int)$element->find('td', 2)->plaintext; $item['leechers'] = (int)$element->find('td', 3)->plaintext; $item['content'] = $element->find('font', 0)->plaintext @@ -163,7 +164,9 @@ class ThePirateBayBridge extends BridgeAbstract { . $item['leechers'] . '
info page'; + . '">info page
magnet link'; if(isset($item['title'])) $this->items[] = $item; diff --git a/bridges/TwitchBridge.php b/bridges/TwitchBridge.php new file mode 100644 index 0000000..39b4601 --- /dev/null +++ b/bridges/TwitchBridge.php @@ -0,0 +1,202 @@ + array( + 'name' => 'Channel', + 'type' => 'text', + 'required' => true, + 'title' => 'Lowercase channel name as seen in channel URL' + ), + 'type' => array( + 'name' => 'Type', + 'type' => 'list', + 'values' => array( + 'All' => 'all', + 'Archive' => 'archive', + 'Highlights' => 'highlight', + 'Uploads' => 'upload' + ), + 'defaultValue' => 'archive' + ) + )); + + /* + * Official instructions for obtaining your own client ID can be found here: + * https://dev.twitch.tv/docs/v5/#getting-a-client-id + */ + const CLIENT_ID = 'kimne78kx3ncx6brgo4mv6wki5h1ko'; + + public function collectData(){ + // get channel user + $query_data = array( + 'login' => $this->getInput('channel') + ); + $users = $this->apiGet('users', $query_data)->users; + if(count($users) === 0) + returnClientError('User "' + . $this->getInput('channel') + . '" could not be found'); + $user = $users[0]; + + // get video list + $query_endpoint = 'channels/' . $user->_id . '/videos'; + $query_data = array( + 'broadcast_type' => $this->getInput('type'), + 'limit' => 10 + ); + $videos = $this->apiGet($query_endpoint, $query_data)->videos; + + foreach($videos as $video) { + $item = array( + 'uri' => $video->url, + 'title' => $video->title, + 'timestamp' => $video->published_at, + 'author' => $video->channel->display_name, + ); + + // Add categories for tags and played game + $item['categories'] = array_filter(explode(' ', $video->tag_list)); + if(!empty($video->game)) + $item['categories'][] = $video->game; + + // Add enclosures for thumbnails from a few points in the video + $item['enclosures'] = array(); + foreach($video->thumbnails->large as $thumbnail) + $item['enclosures'][] = $thumbnail->url; + + /* + * Content format example: + * + * [Preview Image] + * + * Some optional video description. + * + * Duration: 1:23:45 + * Views: 123 + * + * Played games: + * * 00:00:00 Game 1 + * * 00:12:34 Game 2 + * + */ + $item['content'] = '

' + . $video->description_html + . '

Duration: ' + . $this->formatTimestampTime($video->length) + . '
Views: ' + . $video->views + . '

'; + + // Add played games list to content + $video_id = trim($video->_id, 'v'); // _id gives 'v1234' but API wants '1234' + $markers = $this->apiGet('videos/' . $video_id . '/markers')->markers; + $item['content'] .= '

Played games:

  • 00:00:00 - ' + . $video->game + . '
    • '; + if(isset($markers->game_changes)) { + usort($markers->game_changes, function($a, $b) { + return $a->time - $b->time; + }); + foreach($markers->game_changes as $game_change) { + $item['categories'][] = $game_change->label; + $item['content'] .= '
  • ' + . $this->formatTimestampTime($game_change->time) + . ' - ' + . $game_change->label + . '
    • '; + } + } + $item['content'] .= '

'; + + $this->items[] = $item; + } + } + + // e.g. 01:53:27 + private function formatTimestampTime($seconds) { + return sprintf('%02d:%02d:%02d', + floor($seconds / 3600), + ($seconds / 60) % 60, + $seconds % 60); + } + + // e.g. 01h53m27s + private function formatQueryTime($seconds) { + return sprintf('%02dh%02dm%02ds', + floor($seconds / 3600), + ($seconds / 60) % 60, + $seconds % 60); + } + + /* + * Ideally the new 'helix' API should be used as v5/'kraken' is deprecated. + * The new API however still misses many features (markers, played game..) of + * the old one, so let's use the old one for as long as it's available. + */ + private function apiGet($endpoint, $query_data = array()) { + $query_data['api_version'] = 5; + $url = 'https://api.twitch.tv/kraken/' + . $endpoint + . '?' + . http_build_query($query_data); + $header = array( + 'Client-ID: ' . self::CLIENT_ID + ); + + $data = json_decode(getContents($url, $header)) + or returnServerError('API request to "' . $url . '" failed.'); + + return $data; + } + + public function getName(){ + if(!is_null($this->getInput('channel'))) { + return $this->getInput('channel') . ' twitch videos'; + } + + return parent::getName(); + } + + public function getURI(){ + if(!is_null($this->getInput('channel'))) { + return self::URI . $this->getInput('channel'); + } + + return parent::getURI(); + } + + public function detectParameters($url){ + $params = array(); + + // Matches e.g. https://www.twitch.tv/someuser/videos?filter=archives + $regex = '/^(https?:\/\/)? + (www\.)? + twitch\.tv\/ + ([^\/&?\n]+) + \/videos\?.*filter= + (all|archive|highlight|upload)/x'; + if(preg_match($regex, $url, $matches) > 0) { + $params['channel'] = urldecode($matches[3]); + $params['type'] = $matches[4]; + return $params; + } + + return null; + } +} diff --git a/bridges/TwitterBridge.php b/bridges/TwitterBridge.php index 32ed942..2f5565b 100644 --- a/bridges/TwitterBridge.php +++ b/bridges/TwitterBridge.php @@ -28,7 +28,31 @@ class TwitterBridge extends BridgeAbstract { 'name' => 'Keyword or #hashtag', 'required' => true, 'exampleValue' => 'rss-bridge, #rss-bridge', - 'title' => 'Insert a keyword or hashtag' + 'title' => << array( @@ -146,8 +170,15 @@ class TwitterBridge extends BridgeAbstract { public function collectData(){ $html = ''; + $page = $this->getURI(); + + if(php_sapi_name() === 'cli' && empty(ini_get('curl.cainfo'))) { + $cookies = $this->getCookies($page); + $html = getSimpleHTMLDOM($page, array("Cookie: $cookies")); + } else { + $html = getSimpleHTMLDOM($page, array(), array(CURLOPT_COOKIEFILE => '')); + } - $html = getSimpleHTMLDOM($this->getURI()); if(!$html) { switch($this->queriedContext) { case 'By keyword or hashtag': @@ -165,7 +196,7 @@ class TwitterBridge extends BridgeAbstract { // Skip retweets? if($this->getInput('noretweet') - && $tweet->getAttribute('data-screen-name') !== $this->getInput('u')) { + && $tweet->find('div.context span.js-retweet-text a', 0)) { continue; } @@ -189,6 +220,9 @@ class TwitterBridge extends BridgeAbstract { $item['fullname'] = htmlspecialchars_decode($tweet->getAttribute('data-name'), ENT_QUOTES); // get author $item['author'] = $item['fullname'] . ' (@' . $item['username'] . ')'; + if($rt = $tweet->find('div.context span.js-retweet-text a', 0)) { + $item['author'] .= ' RT: @' . $rt->plaintext; + } // get avatar link $item['avatar'] = $tweet->find('img', 0)->src; // get TweetID @@ -242,22 +276,26 @@ EOD; // Add embeded image to content $image_html = ''; - $image = $this->getImageURI($tweet); - if(!$this->getInput('noimg') && !is_null($image)) { - // Set image scaling - $image_orig = $this->getInput('noimgscaling') ? $image : $image . ':orig'; - $image_thumb = $this->getInput('noimgscaling') ? $image : $image . ':thumb'; + $images = $this->getImageURI($tweet); + if(!$this->getInput('noimg') && !is_null($images)) { + + foreach ($images as $image) { - // add enclosures - $item['enclosures'] = array($image_orig); + // Set image scaling + $image_orig = $this->getInput('noimgscaling') ? $image : $image . ':orig'; + $image_thumb = $this->getInput('noimgscaling') ? $image : $image . ':thumb'; + + // add enclosures + $item['enclosures'][] = $image_orig; - $image_html = << EOD; + } } // add content @@ -288,22 +326,27 @@ EOD; // Add embeded image to content $quotedImage_html = ''; - $quotedImage = $this->getQuotedImageURI($tweet); - if(!$this->getInput('noimg') && !is_null($quotedImage)) { - // Set image scaling - $quotedImage_orig = $this->getInput('noimgscaling') ? $quotedImage : $quotedImage . ':orig'; - $quotedImage_thumb = $this->getInput('noimgscaling') ? $quotedImage : $quotedImage . ':thumb'; + $quotedImages = $this->getQuotedImageURI($tweet); - // add enclosures - $item['enclosures'] = array($quotedImage_orig); + if(!$this->getInput('noimg') && !is_null($quotedImages)) { + + foreach ($quotedImages as $image) { + + // Set image scaling + $image_orig = $this->getInput('noimgscaling') ? $image : $image . ':orig'; + $image_thumb = $this->getInput('noimgscaling') ? $image : $image . ':thumb'; - $quotedImage_html = << + // add enclosures + $item['enclosures'][] = $image_orig; + + $quotedImage_html .= << + src="{$image_thumb}" /> EOD; + } } $item['content'] = <<find('div.AdaptiveMedia-container', 0); + if($container && $container->find('img', 0)) { - return $container->find('img', 0)->src; + foreach ($container->find('img') as $img) { + $images[] = $img->src; + } + } + + if (!empty($images)) { + return $images; } return null; @@ -367,11 +419,43 @@ EOD; private function getQuotedImageURI($tweet){ // Find media in tweet + $images = array(); + $container = $tweet->find('div.QuoteMedia-container', 0); + if($container && $container->find('img', 0)) { - return $container->find('img', 0)->src; + foreach ($container->find('img') as $img) { + $images[] = $img->src; + } + } + + if (!empty($images)) { + return $images; } return null; } + + private function getCookies($pageURL){ + + $ctx = stream_context_create(array( + 'http' => array( + 'follow_location' => false + ) + ) + ); + $a = file_get_contents($pageURL, 0, $ctx); + + //First request to get the cookie + $cookies = ''; + foreach($http_response_header as $hdr) { + if(stripos($hdr, 'Set-Cookie') !== false) { + $cLine = explode(':', $hdr)[1]; + $cLine = explode(';', $cLine)[0]; + $cookies .= ';' . $cLine; + } + } + + return substr($cookies, 2); + } } diff --git a/bridges/UnsplashBridge.php b/bridges/UnsplashBridge.php index ae76734..dad0efc 100644 --- a/bridges/UnsplashBridge.php +++ b/bridges/UnsplashBridge.php @@ -3,7 +3,7 @@ class UnsplashBridge extends BridgeAbstract { const MAINTAINER = 'nel50n'; const NAME = 'Unsplash Bridge'; - const URI = 'http://unsplash.com/'; + const URI = 'https://unsplash.com/'; const CACHE_TIMEOUT = 43200; // 12h const DESCRIPTION = 'Returns the latests photos from Unsplash'; @@ -27,51 +27,42 @@ class UnsplashBridge extends BridgeAbstract { public function collectData(){ $width = $this->getInput('w'); - $num = 0; $max = $this->getInput('m'); $quality = $this->getInput('q'); - $lastpage = 1; - for($page = 1; $page <= $lastpage; $page++) { - $link = self::URI . '/grid?page=' . $page; - $html = getSimpleHTMLDOM($link) - or returnServerError('No results for this query.'); + $api_response = getContents('https://unsplash.com/napi/photos?page=1&per_page=' . $max) + or returnServerError('Could not request Unsplash API.'); + $json = json_decode($api_response, true); - if($page === 1) { - preg_match( - '/=(\d+)$/', - $html->find('.pagination > a[!class]', -1)->href, - $matches - ); + foreach ($json as $json_item) { + $item = array(); - $lastpage = min($matches[1], ceil($max / 40)); - } - - foreach($html->find('.photo') as $element) { - $thumbnail = $element->find('img', 0); - $thumbnail->src = str_replace('https://', 'http://', $thumbnail->src); + // Get image URI + $uri = $json_item['urls']['regular'] . '.jpg'; // '.jpg' only for format hint + $uri = str_replace('q=80', 'q=' . $quality, $uri); + $uri = str_replace('w=1080', 'w=' . $width, $uri); + $item['uri'] = $uri; - $item = array(); - $item['uri'] = str_replace( - array('q=75', 'w=400'), - array("q=$quality", "w=$width"), - $thumbnail->src) . '.jpg'; // '.jpg' only for format hint + // Get title from description + if (is_null($json_item['alt_description'])) { + if (is_null($json_item['description'])) { + $item['title'] = 'Unsplash picture from ' . $json_item['user']['name']; + } else { + $item['title'] = $json_item['description']; + } + } else { + $item['title'] = $json_item['alt_description']; + } - $item['timestamp'] = time(); - $item['title'] = $thumbnail->alt; - $item['content'] = $item['title'] + $item['timestamp'] = time(); + $item['content'] = $item['title'] . '
'; - $this->items[] = $item; - - $num++; - if ($num >= $max) - break 2; - } + $this->items[] = $item; } } } diff --git a/bridges/VMwareSecurityBridge.php b/bridges/VMwareSecurityBridge.php new file mode 100644 index 0000000..326d26a --- /dev/null +++ b/bridges/VMwareSecurityBridge.php @@ -0,0 +1,31 @@ +find('div[class="news_block"]'); + + foreach ($articles as $element) { + $item['uri'] = $element->find('a', 0)->getAttribute('href'); + $title = $element->find('a', 0)->innertext; + $item['title'] = $title; + $item['timestamp'] = strtotime($element->find('p', 0)->innertext); + $item['content'] = $element->find('p', 1)->innertext; + $item['uid'] = $title; + + $this->items[] = $item; + } + } +} diff --git a/bridges/VimeoBridge.php b/bridges/VimeoBridge.php new file mode 100644 index 0000000..d318e30 --- /dev/null +++ b/bridges/VimeoBridge.php @@ -0,0 +1,175 @@ + array( + 'name' => 'Search Query', + 'type' => 'text', + 'required' => true + ), + 'type' => array( + 'name' => 'Show results for', + 'type' => 'list', + 'defaultValue' => 'Videos', + 'values' => array( + 'Videos' => 'search', + 'On Demand' => 'search/ondemand', + 'People' => 'search/people', + 'Channels' => 'search/channels', + 'Groups' => 'search/groups' + ) + ) + ) + ); + + public function getURI() { + if(($query = $this->getInput('q')) + && ($type = $this->getInput('type'))) { + return self::URI . $type . '/sort:latest?q=' . $query; + } + + return parent::getURI(); + } + + public function collectData() { + + $html = getSimpleHTMLDOM($this->getURI(), + $header = array(), + $opts = array(), + $lowercase = true, + $forceTagsClosed = true, + $target_charset = DEFAULT_TARGET_CHARSET, + $stripRN = false, // We want to keep newline characters + $defaultBRText = DEFAULT_BR_TEXT, + $defaultSpanText = DEFAULT_SPAN_TEXT) + or returnServerError('Could not request ' . $this->getURI()); + + $json = null; // Holds the JSON data + + /** + * Search results are included as JSON formatted string inside a script + * tag that has the variable 'vimeo.config'. The data is condensed into + * a single line of code, so we can just search for the newline. + * + * Everything after "vimeo.config = _extend((vimeo.config || {}), " is + * the JSON formatted string. + */ + foreach($html->find('script') as $script) { + foreach(explode("\n", $script) as $line) { + $line = trim($line); + + if(strpos($line, 'vimeo.config') !== 0) + continue; + + // 45 = strlen("vimeo.config = _extend((vimeo.config || {}), "); + // 47 = 45 + 2, because we don't want the final ");" + $json = json_decode(substr($line, 45, strlen($line) - 47)); + } + } + + if(is_null($json)) { + returnClientError('No results for this query!'); + } + + foreach($json->api->initial_json->data as $element) { + switch($element->type) { + case 'clip': $this->addClip($element); break; + case 'ondemand': $this->addOnDemand($element); break; + case 'people': $this->addPeople($element); break; + case 'channel': $this->addChannel($element); break; + case 'group': $this->addGroup($element); break; + + default: returnServerError('Unknown type: ' . $element->type); + } + } + + } + + private function addClip($element) { + $item = array(); + + $item['uri'] = $element->clip->link; + $item['title'] = $element->clip->name; + $item['author'] = $element->clip->user->name; + $item['timestamp'] = strtotime($element->clip->created_time); + + $item['enclosures'] = array( + end($element->clip->pictures->sizes)->link + ); + + $item['content'] = ""; + + $this->items[] = $item; + } + + private function addOnDemand($element) { + $item = array(); + + $item['uri'] = $element->ondemand->link; + $item['title'] = $element->ondemand->name; + + // Only for films + if(isset($element->ondemand->film)) + $item['timestamp'] = strtotime($element->ondemand->film->release_time); + + $item['enclosures'] = array( + end($element->ondemand->pictures->sizes)->link + ); + + $item['content'] = ""; + + $this->items[] = $item; + } + + private function addPeople($element) { + $item = array(); + + $item['uri'] = $element->people->link; + $item['title'] = $element->people->name; + + $item['enclosures'] = array( + end($element->people->pictures->sizes)->link + ); + + $item['content'] = ""; + + $this->items[] = $item; + } + + private function addChannel($element) { + $item = array(); + + $item['uri'] = $element->channel->link; + $item['title'] = $element->channel->name; + + $item['enclosures'] = array( + end($element->channel->pictures->sizes)->link + ); + + $item['content'] = ""; + + $this->items[] = $item; + } + + private function addGroup($element) { + $item = array(); + + $item['uri'] = $element->group->link; + $item['title'] = $element->group->name; + + $item['enclosures'] = array( + end($element->group->pictures->sizes)->link + ); + + $item['content'] = ""; + + $this->items[] = $item; + } +} diff --git a/bridges/VkBridge.php b/bridges/VkBridge.php index d4e84d9..f9aaa66 100644 --- a/bridges/VkBridge.php +++ b/bridges/VkBridge.php @@ -13,6 +13,10 @@ class VkBridge extends BridgeAbstract 'u' => array( 'name' => 'Group or user name', 'required' => true + ), + 'hide_reposts' => array( + 'name' => 'Hide reposts', + 'type' => 'checkbox', ) ) ); @@ -48,7 +52,7 @@ class VkBridge extends BridgeAbstract $text_html = $this->getContents() or returnServerError('No results for group or user name "' . $this->getInput('u') . '".'); - $text_html = iconv('windows-1251', 'utf-8', $text_html); + $text_html = iconv('windows-1251', 'utf-8//ignore', $text_html); // makes album link generating work correctly $text_html = str_replace('"class="page_album_link">', '" class="page_album_link">', $text_html); $html = str_get_html($text_html); @@ -234,6 +238,9 @@ class VkBridge extends BridgeAbstract } if (is_object($post->find('div.copy_quote', 0))) { + if ($this->getInput('hide_reposts') === true) { + continue; + } $copy_quote = $post->find('div.copy_quote', 0); if ($copy_post_header = $copy_quote->find('div.copy_post_header', 0)) { $copy_post_header->outertext = ''; diff --git a/bridges/WikiLeaksBridge.php b/bridges/WikiLeaksBridge.php index c5b9bb6..363cf0c 100644 --- a/bridges/WikiLeaksBridge.php +++ b/bridges/WikiLeaksBridge.php @@ -9,7 +9,6 @@ class WikiLeaksBridge extends BridgeAbstract { 'category' => array( 'name' => 'Category', 'type' => 'list', - 'required' => true, 'title' => 'Select your category', 'values' => array( 'News' => '-News-', @@ -28,7 +27,6 @@ class WikiLeaksBridge extends BridgeAbstract { 'teaser' => array( 'name' => 'Show teaser', 'type' => 'checkbox', - 'required' => false, 'title' => 'If checked feeds will display the teaser', 'defaultValue' => true ) diff --git a/bridges/WikipediaBridge.php b/bridges/WikipediaBridge.php index 6b53440..7ca763f 100644 --- a/bridges/WikipediaBridge.php +++ b/bridges/WikipediaBridge.php @@ -13,7 +13,6 @@ class WikipediaBridge extends BridgeAbstract { 'language' => array( 'name' => 'Language', 'type' => 'list', - 'required' => true, 'title' => 'Select your language', 'exampleValue' => 'English', 'values' => array( @@ -27,7 +26,6 @@ class WikipediaBridge extends BridgeAbstract { 'subject' => array( 'name' => 'Subject', 'type' => 'list', - 'required' => true, 'title' => 'What subject are you interested in?', 'exampleValue' => 'Today\'s featured article', 'values' => array( diff --git a/bridges/WiredBridge.php b/bridges/WiredBridge.php new file mode 100644 index 0000000..8da93d0 --- /dev/null +++ b/bridges/WiredBridge.php @@ -0,0 +1,102 @@ + array( + 'name' => 'Feed', + 'type' => 'list', + 'values' => array( + 'WIRED Top Stories' => 'rss', // /feed/rss + 'Business' => 'business', // /feed/category/business/latest/rss + 'Culture' => 'culture', // /feed/category/culture/latest/rss + 'Gear' => 'gear', // /feed/category/gear/latest/rss + 'Ideas' => 'ideas', // /feed/category/ideas/latest/rss + 'Science' => 'science', // /feed/category/science/latest/rss + 'Security' => 'security', // /feed/category/security/latest/rss + 'Transportation' => 'transportation', // /feed/category/transportation/latest/rss + 'Backchannel' => 'backchannel', // /feed/category/backchannel/latest/rss + 'WIRED Guides' => 'wired-guide', // /feed/tag/wired-guide/latest/rss + 'Photo' => 'photo' // /feed/category/photo/latest/rss + ) + ) + )); + + public function collectData(){ + $feed = $this->getInput('feed'); + if(empty($feed) || !ctype_alpha(str_replace('-', '', $feed))) { + returnClientError('Invalid feed, please check the "feed" parameter.'); + } + + $feed_url = $this->getURI() . 'feed/'; + if ($feed != 'rss') { + if ($feed != 'wired-guide') { + $feed_url .= 'category/'; + } else { + $feed_url .= 'tag/'; + } + $feed_url .= "$feed/latest/"; + } + $feed_url .= 'rss'; + + $this->collectExpandableDatas($feed_url); + } + + protected function parseItem($newsItem){ + $item = parent::parseItem($newsItem); + $article = getSimpleHTMLDOMCached($item['uri']) + or returnServerError('Could not request WIRED: ' . $item['uri']); + $item['content'] = $this->extractArticleContent($article); + + $headline = strval($newsItem->description); + if(!empty($headline)) { + $item['content'] = '

' . $headline . '

' . $item['content']; + } + + $item_image = $article->find('meta[property="og:image"]', 0); + if(!empty($item_image)) { + $item['enclosures'] = array($item_image->content); + $item['content'] = '

' . $item['content']; + } + + return $item; + } + + private function extractArticleContent($article){ + $content = $article->find('article', 0); + $truncate = true; + + if (empty($content)) { + $content = $article->find('div.listicle-main-component__container', 0); + $truncate = false; + } + + if (!empty($content)) { + $content = $content->innertext; + } + + foreach (array( + '
find('div[class~="p-body"]', 0)) { $this->version = self::XENFORO_VERSION_2; } else { returnServerError('This forum is currently not supported!'); @@ -127,7 +127,7 @@ class XenForoBridge extends BridgeAbstract { switch($this->version) { case self::XENFORO_VERSION_1: - $titleBar = $mainContent->find('div.titleBar h1', 0) + $titleBar = $mainContent->find('div.titleBar > h1', 0) or returnServerError('Error finding title bar!'); $this->title = $titleBar->plaintext; @@ -140,7 +140,7 @@ class XenForoBridge extends BridgeAbstract { case self::XENFORO_VERSION_2: - $titleBar = $mainContent->find('div[class="p-title"] h1', 0) + $titleBar = $mainContent->find('div[class~="p-title"] h1', 0) or returnServerError('Error finding title bar!'); $this->title = $titleBar->plaintext; @@ -166,7 +166,7 @@ class XenForoBridge extends BridgeAbstract { $lang = $html->find('html', 0)->lang; // Posts are contained in an "ol" - $messageList = $html->find('#messageList li') + $messageList = $html->find('#messageList > li') or returnServerError('Error finding message list!'); foreach($messageList as $post) { @@ -179,7 +179,7 @@ class XenForoBridge extends BridgeAbstract { $item['uri'] = $url . '#' . $post->getAttribute('id'); - $content = $post->find('.messageContent article', 0); + $content = $post->find('.messageContent > article', 0); // Add some style to quotes foreach($content->find('.bbCodeQuote') as $quote) { @@ -255,7 +255,7 @@ class XenForoBridge extends BridgeAbstract { $lang = $html->find('html', 0)->lang; - $messageList = $html->find('div[class="block-body"] article') + $messageList = $html->find('div[class~="block-body"] article') or returnServerError('Error finding message list!'); foreach($messageList as $post) { @@ -268,13 +268,17 @@ class XenForoBridge extends BridgeAbstract { $item['uri'] = $url . '#' . $post->getAttribute('id'); - $title = $post->find('div[class="message-content"] article', 0)->plaintext; + $title = $post->find('div[class~="message-content"] article', 0)->plaintext; $end = strpos($title, ' ', 70); $item['title'] = substr($title, 0, $end); - $item['timestamp'] = $this->fixDate($post->find('time', 0)->title, $lang); + if ($post->find('time[datetime]', 0)) { + $item['timestamp'] = $post->find('time[datetime]', 0)->datetime; + } else { + $item['timestamp'] = $this->fixDate($post->find('time', 0)->title, $lang); + } $item['author'] = $post->getAttribute('data-author'); - $item['content'] = $post->find('div[class="message-content"] article', 0); + $item['content'] = $post->find('div[class~="message-content"] article', 0); // Bridge specific properties $item['id'] = $post->getAttribute('id'); @@ -305,7 +309,7 @@ class XenForoBridge extends BridgeAbstract { // Load at least the last page do { - $pageurl = $hosturl . str_replace($sentinel, $lastpage, $baseurl); + $pageurl = str_replace($sentinel, $lastpage, $baseurl); // We can optimize performance by caching all but the last page if($page != $lastpage) { @@ -339,7 +343,7 @@ class XenForoBridge extends BridgeAbstract { } // Manually extract baseurl and inject sentinel - $baseurl = $pageNav->find('li a', -1)->href; + $baseurl = $pageNav->find('li > a', -1)->href; $baseurl = str_replace('page-' . $lastpage, 'page-{{sentinel}}', $baseurl); $sentinel = '{{sentinel}}'; @@ -353,7 +357,7 @@ class XenForoBridge extends BridgeAbstract { // Load at least the last page do { - $pageurl = $hosturl . str_replace($sentinel, $lastpage, $baseurl); + $pageurl = str_replace($sentinel, $lastpage, $baseurl); // We can optimize performance by caching all but the last page if($page != $lastpage) { @@ -364,9 +368,9 @@ class XenForoBridge extends BridgeAbstract { or returnServerError('Error loading contents from ' . $pageurl . '!'); } - $html = defaultLinkTo($html, $this->hosturl); + $html = defaultLinkTo($html, $hosturl); - $this->extractThreadPostsV2($html, $this->pageurl); + $this->extractThreadPostsV2($html, $pageurl); $page--; diff --git a/bridges/YoutubeBridge.php b/bridges/YoutubeBridge.php index 67e9566..90ee049 100644 --- a/bridges/YoutubeBridge.php +++ b/bridges/YoutubeBridge.php @@ -65,7 +65,7 @@ class YoutubeBridge extends BridgeAbstract { private $feedName = ''; private function ytBridgeQueryVideoInfo($vid, &$author, &$desc, &$time){ - $html = $this->ytGetSimpleHTMLDOM(self::URI . "watch?v=$vid"); + $html = $this->ytGetSimpleHTMLDOM(self::URI . "watch?v=$vid", true); // Skip unavailable videos if(!strpos($html->innertext, 'IS_UNAVAILABLE_PAGE')) { @@ -127,7 +127,6 @@ class YoutubeBridge extends BridgeAbstract { } private function ytBridgeParseHtmlListing($html, $element_selector, $title_selector, $add_parsed_items = true) { - $limit = $add_parsed_items ? 10 : INF; $count = 0; $duration_min = $this->getInput('duration_min') ?: -1; @@ -141,40 +140,38 @@ class YoutubeBridge extends BridgeAbstract { } foreach($html->find($element_selector) as $element) { - if($count < $limit) { - $author = ''; - $desc = ''; - $time = 0; - $vid = str_replace('/watch?v=', '', $element->find('a', 0)->href); - $vid = substr($vid, 0, strpos($vid, '&') ?: strlen($vid)); - $title = trim($this->ytBridgeFixTitle($element->find($title_selector, 0)->plaintext)); - - if (strpos($vid, 'googleads') !== false - || $title == '[Private video]' - || $title == '[Deleted video]' - ) { - continue; - } - - // The duration comes in one of the formats: - // hh:mm:ss / mm:ss / m:ss - // 01:03:30 / 15:06 / 1:24 - $durationText = trim($element->find('div.timestamp span', 0)->plaintext); - $durationText = preg_replace('/([\d]{1,2})\:([\d]{2})/', '00:$1:$2', $durationText); - - sscanf($durationText, '%d:%d:%d', $hours, $minutes, $seconds); - $duration = $hours * 3600 + $minutes * 60 + $seconds; - - if($duration < $duration_min || $duration > $duration_max) { - continue; - } - - if ($add_parsed_items) { - $this->ytBridgeQueryVideoInfo($vid, $author, $desc, $time); - $this->ytBridgeAddItem($vid, $title, $author, $desc, $time); - } - $count++; + $author = ''; + $desc = ''; + $time = 0; + $vid = str_replace('/watch?v=', '', $element->find('a', 0)->href); + $vid = substr($vid, 0, strpos($vid, '&') ?: strlen($vid)); + $title = trim($this->ytBridgeFixTitle($element->find($title_selector, 0)->plaintext)); + + if (strpos($vid, 'googleads') !== false + || $title == '[Private video]' + || $title == '[Deleted video]' + ) { + continue; } + + // The duration comes in one of the formats: + // hh:mm:ss / mm:ss / m:ss + // 01:03:30 / 15:06 / 1:24 + $durationText = trim($element->find('div.timestamp span', 0)->plaintext); + $durationText = preg_replace('/([\d]{1,2})\:([\d]{2})/', '00:$1:$2', $durationText); + + sscanf($durationText, '%d:%d:%d', $hours, $minutes, $seconds); + $duration = $hours * 3600 + $minutes * 60 + $seconds; + + if($duration < $duration_min || $duration > $duration_max) { + continue; + } + + if ($add_parsed_items) { + $this->ytBridgeQueryVideoInfo($vid, $author, $desc, $time); + $this->ytBridgeAddItem($vid, $title, $author, $desc, $time); + } + $count++; } return $count; } @@ -184,18 +181,38 @@ class YoutubeBridge extends BridgeAbstract { return html_entity_decode($title, ENT_QUOTES, 'UTF-8'); } - private function ytGetSimpleHTMLDOM($url){ + private function ytGetSimpleHTMLDOM($url, $cached = false){ + $header = array( + 'Accept-Language: en-US' + ); + $opts = array(); + $lowercase = true; + $forceTagsClosed = true; + $target_charset = DEFAULT_TARGET_CHARSET; + $stripRN = false; + $defaultBRText = DEFAULT_BR_TEXT; + $defaultSpanText = DEFAULT_SPAN_TEXT; + if ($cached) { + return getSimpleHTMLDOMCached($url, + 86400, + $header, + $opts, + $lowercase, + $forceTagsClosed, + $target_charset, + $stripRN, + $defaultBRText, + $defaultSpanText); + } return getSimpleHTMLDOM($url, - $header = array( - 'Accept-Language: en-US' - ), - $opts = array(), - $lowercase = true, - $forceTagsClosed = true, - $target_charset = DEFAULT_TARGET_CHARSET, - $stripRN = false, - $defaultBRText = DEFAULT_BR_TEXT, - $defaultSpanText = DEFAULT_SPAN_TEXT); + $header, + $opts, + $lowercase, + $forceTagsClosed, + $target_charset, + $stripRN, + $defaultBRText, + $defaultSpanText); } public function collectData(){ diff --git a/cache/pages/.gitkeep b/cache/pages/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/cache/server/.gitkeep b/cache/server/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/caches/FileCache.php b/caches/FileCache.php index 04d08a2..166ecdb 100644 --- a/caches/FileCache.php +++ b/caches/FileCache.php @@ -3,20 +3,21 @@ * Cache with file system */ class FileCache implements CacheInterface { - protected $path; - protected $param; + protected $key; public function loadData(){ if(file_exists($this->getCacheFile())) { return unserialize(file_get_contents($this->getCacheFile())); } + + return null; } - public function saveData($datas){ + public function saveData($data){ // Notice: We use plain serialize() here to reduce memory footprint on // large input data. - $writeStream = file_put_contents($this->getCacheFile(), serialize($datas)); + $writeStream = file_put_contents($this->getCacheFile(), serialize($data)); if($writeStream === false) { throw new \Exception('Cannot write the cache... Do you have the right permissions ?'); @@ -29,13 +30,14 @@ class FileCache implements CacheInterface { $cacheFile = $this->getCacheFile(); clearstatcache(false, $cacheFile); if(file_exists($cacheFile)) { - return filemtime($cacheFile); + $time = filemtime($cacheFile); + return ($time !== false) ? $time : null; } - return false; + return null; } - public function purgeCache($duration){ + public function purgeCache($seconds){ $cachePath = $this->getPath(); if(file_exists($cachePath)) { $cacheIterator = new RecursiveIteratorIterator( @@ -47,7 +49,7 @@ class FileCache implements CacheInterface { if(in_array($cacheFile->getBasename(), array('.', '..', '.gitkeep'))) continue; elseif($cacheFile->isFile()) { - if(filemtime($cacheFile->getPathname()) < time() - $duration) + if(filemtime($cacheFile->getPathname()) < time() - $seconds) unlink($cacheFile->getPathname()); } } @@ -55,34 +57,34 @@ class FileCache implements CacheInterface { } /** - * Set cache path + * Set scope * @return self */ - public function setPath($path){ - if(is_null($path) || !is_string($path)) { - throw new \Exception('The given path is invalid!'); + public function setScope($scope){ + if(is_null($scope) || !is_string($scope)) { + throw new \Exception('The given scope is invalid!'); } - $this->path = $path; - - // Make sure path ends with '/' or '\' - $lastchar = substr($this->path, -1, 1); - if($lastchar !== '/' && $lastchar !== '\\') - $this->path .= '/'; - - if(!is_dir($this->path)) - mkdir($this->path, 0755, true); + $this->path = PATH_CACHE . trim($scope, " \t\n\r\0\x0B\\\/") . '/'; return $this; } /** - * Set HTTP GET parameters + * Set key * @return self */ - public function setParameters(array $param){ - $this->param = array_map('strtolower', $param); + public function setKey($key){ + if (!empty($key) && is_array($key)) { + $key = array_map('strtolower', $key); + } + $key = json_encode($key); + if (!is_string($key)) { + throw new \Exception('The given key is invalid!'); + } + + $this->key = $key; return $this; } @@ -90,9 +92,15 @@ class FileCache implements CacheInterface { * Return cache path (and create if not exist) * @return string Cache path */ - protected function getPath(){ + private function getPath(){ if(is_null($this->path)) { - throw new \Exception('Call "setPath" first!'); + throw new \Exception('Call "setScope" first!'); + } + + if(!is_dir($this->path)) { + if (mkdir($this->path, 0755, true) !== true) { + throw new \Exception('Unable to create ' . $this->path); + } } return $this->path; @@ -102,7 +110,7 @@ class FileCache implements CacheInterface { * Get the file name use for cache store * @return string Path to the file cache */ - protected function getCacheFile(){ + private function getCacheFile(){ return $this->getPath() . $this->getCacheName(); } @@ -110,13 +118,11 @@ class FileCache implements CacheInterface { * Determines file name for store the cache * return string */ - protected function getCacheName(){ - if(is_null($this->param)) { - throw new \Exception('Call "setParameters" first!'); + private function getCacheName(){ + if(is_null($this->key)) { + throw new \Exception('Call "setKey" first!'); } - // Change character when making incompatible changes to prevent loading - // errors due to incompatible file contents \|/ - return hash('md5', http_build_query($this->param) . 'A') . '.cache'; + return hash('md5', $this->key) . '.cache'; } } diff --git a/caches/MemcachedCache.php b/caches/MemcachedCache.php new file mode 100644 index 0000000..f69f10b --- /dev/null +++ b/caches/MemcachedCache.php @@ -0,0 +1,115 @@ + 65535) { + returnServerError('"port" param is invalid for ' . get_called_class() . '. Please check your ' . FILE_CONFIG); + } + + $conn = new Memcached(); + $conn->addServer($host, $port) or returnServerError('Could not connect to memcached server'); + $this->conn = $conn; + } + + public function loadData(){ + if ($this->data) return $this->data; + $result = $this->conn->get($this->getCacheKey()); + if ($result === false) { + return false; + } + + $this->time = $result['time']; + $this->data = $result['data']; + return $result['data']; + } + + public function saveData($datas){ + $time = time(); + $object_to_save = array( + 'data' => $datas, + 'time' => $time, + ); + $result = $this->conn->set($this->getCacheKey(), $object_to_save, $this->expiration); + + if($result === false) { + returnServerError('Cannot write the cache to memcached server'); + } + + $this->time = $time; + + return $this; + } + + public function getTime(){ + if ($this->time === false) { + $this->loadData(); + } + return $this->time; + } + + public function purgeCache($duration){ + // Note: does not purges cache right now + // Just sets cache expiration and leave cache purging for memcached itself + $this->expiration = $duration; + } + + /** + * Set scope + * @return self + */ + public function setScope($scope){ + $this->scope = $scope; + return $this; + } + + /** + * Set key + * @return self + */ + public function setKey($key){ + if (!empty($key) && is_array($key)) { + $key = array_map('strtolower', $key); + } + $key = json_encode($key); + + if (!is_string($key)) { + throw new \Exception('The given key is invalid!'); + } + + $this->key = $key; + return $this; + } + + private function getCacheKey(){ + if(is_null($this->key)) { + returnServerError('Call "setKey" first!'); + } + + return 'rss_bridge_cache_' . hash('md5', $this->scope . $this->key . 'A'); + } +} diff --git a/caches/SQLiteCache.php b/caches/SQLiteCache.php new file mode 100644 index 0000000..394e25f --- /dev/null +++ b/caches/SQLiteCache.php @@ -0,0 +1,121 @@ + + */ +class SQLiteCache implements CacheInterface { + protected $scope; + protected $key; + + private $db = null; + + public function __construct() { + if (!extension_loaded('sqlite3')) { + die('"sqlite3" extension not loaded. Please check "php.ini"'); + } + + $file = Configuration::getConfig(get_called_class(), 'file'); + if (empty($file)) { + die('Configuration for ' . get_called_class() . ' missing. Please check your ' . FILE_CONFIG); + } + if (dirname($file) == '.') { + $file = PATH_CACHE . $file; + } elseif (!is_dir(dirname($file))) { + die('Invalid configuration for ' . get_called_class() . '. Please check your ' . FILE_CONFIG); + } + + if (!is_file($file)) { + $this->db = new SQLite3($file); + $this->db->enableExceptions(true); + $this->db->exec("CREATE TABLE storage ('key' BLOB PRIMARY KEY, 'value' BLOB, 'updated' INTEGER)"); + } else { + $this->db = new SQLite3($file); + $this->db->enableExceptions(true); + } + $this->db->busyTimeout(5000); + } + + public function loadData(){ + $Qselect = $this->db->prepare('SELECT value FROM storage WHERE key = :key'); + $Qselect->bindValue(':key', $this->getCacheKey()); + $result = $Qselect->execute(); + if ($result instanceof SQLite3Result) { + $data = $result->fetchArray(SQLITE3_ASSOC); + if (isset($data['value'])) { + return unserialize($data['value']); + } + } + + return null; + } + + public function saveData($data){ + $Qupdate = $this->db->prepare('INSERT OR REPLACE INTO storage (key, value, updated) VALUES (:key, :value, :updated)'); + $Qupdate->bindValue(':key', $this->getCacheKey()); + $Qupdate->bindValue(':value', serialize($data)); + $Qupdate->bindValue(':updated', time()); + $Qupdate->execute(); + + return $this; + } + + public function getTime(){ + $Qselect = $this->db->prepare('SELECT updated FROM storage WHERE key = :key'); + $Qselect->bindValue(':key', $this->getCacheKey()); + $result = $Qselect->execute(); + if ($result instanceof SQLite3Result) { + $data = $result->fetchArray(SQLITE3_ASSOC); + if (isset($data['updated'])) { + return $data['updated']; + } + } + + return null; + } + + public function purgeCache($seconds){ + $Qdelete = $this->db->prepare('DELETE FROM storage WHERE updated < :expired'); + $Qdelete->bindValue(':expired', time() - $seconds); + $Qdelete->execute(); + } + + /** + * Set scope + * @return self + */ + public function setScope($scope){ + if(is_null($scope) || !is_string($scope)) { + throw new \Exception('The given scope is invalid!'); + } + + $this->scope = $scope; + return $this; + } + + /** + * Set key + * @return self + */ + public function setKey($key){ + if (!empty($key) && is_array($key)) { + $key = array_map('strtolower', $key); + } + $key = json_encode($key); + + if (!is_string($key)) { + throw new \Exception('The given key is invalid!'); + } + + $this->key = $key; + return $this; + } + + //////////////////////////////////////////////////////////////////////////// + + private function getCacheKey(){ + if(is_null($this->key)) { + throw new \Exception('Call "setKey" first!'); + } + + return hash('sha1', $this->scope . $this->key, true); + } +} diff --git a/composer.json b/composer.json new file mode 100644 index 0000000..8748cb3 --- /dev/null +++ b/composer.json @@ -0,0 +1,12 @@ +{ + "require": { + "php": ">=5.6", + "ext-mbstring": "*", + "ext-sqlite3": "*", + "ext-curl": "*", + "ext-openssl": "*", + "ext-libxml": "*", + "ext-simplexml": "*", + "ext-json": "*" + } +} diff --git a/composer.lock b/composer.lock new file mode 100644 index 0000000..3d8d9f2 --- /dev/null +++ b/composer.lock @@ -0,0 +1,26 @@ +{ + "_readme": [ + "This file locks the dependencies of your project to a known state", + "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", + "This file is @generated automatically" + ], + "content-hash": "ef341ee18f28c7bd5832e188fe157734", + "packages": [], + "packages-dev": [], + "aliases": [], + "minimum-stability": "stable", + "stability-flags": [], + "prefer-stable": false, + "prefer-lowest": false, + "platform": { + "php": ">=5.6", + "ext-mbstring": "*", + "ext-sqlite3": "*", + "ext-curl": "*", + "ext-openssl": "*", + "ext-libxml": "*", + "ext-simplexml": "*", + "ext-json": "*" + }, + "platform-dev": [] +} diff --git a/config.default.ini.php b/config.default.ini.php index 2d6fca1..b7d4fba 100644 --- a/config.default.ini.php +++ b/config.default.ini.php @@ -4,8 +4,20 @@ ; file, it will be replaced on the next update of RSS-Bridge! You can specify ; your own configuration in 'config.ini.php' (copy this file). +[system] + +; Defines the timezone used by RSS-Bridge +; Find a list of supported timezones at +; https://www.php.net/manual/en/timezones.php +; timezone = "UTC" (default) +timezone = "UTC" + [cache] +; Defines the cache type used by RSS-Bridge +; "file" = FileCache (default) +type = "file" + ; Allow users to specify custom timeout for specific requests. ; true = enabled ; false = disabled (default) @@ -48,3 +60,12 @@ username = "" ; The password for authentication. Insert this password when prompted for login. ; Use a strong password to prevent others from guessing your login! password = "" + +; --- Cache specific configuration --------------------------------------------- + +[SQLiteCache] +file = "cache.sqlite" + +[MemcachedCache] +host = "localhost" +port = 11211 diff --git a/formats/AtomFormat.php b/formats/AtomFormat.php index bb5e30e..1159a61 100644 --- a/formats/AtomFormat.php +++ b/formats/AtomFormat.php @@ -1,21 +1,30 @@ xml_encode($_SERVER['REQUEST_URI']) : ''; + $feedUrl = $this->xml_encode($urlPrefix . $urlHost . $urlRequest); $extraInfos = $this->getExtraInfos(); $title = $this->xml_encode($extraInfos['name']); $uri = !empty($extraInfos['uri']) ? $extraInfos['uri'] : REPOSITORY; + // since we can't guarantee that all items have an author, + // a global feed author is mandatory + $feedAuthor = 'RSS-Bridge'; + $uriparts = parse_url($uri); if(!empty($extraInfos['icon'])) { $icon = $extraInfos['icon']; @@ -27,11 +36,40 @@ class AtomFormat extends FormatAbstract{ $entries = ''; foreach($this->getItems() as $item) { + $entryTimestamp = $item->getTimestamp(); + $entryTitle = $item->getTitle(); + $entryContent = $item->getContent(); + $entryUri = $item->getURI(); + $entryID = ''; + + if (!empty($item->getUid())) + $entryID = 'urn:sha1:' . $item->getUid(); + + if (empty($entryID)) // Fallback to provided URI + $entryID = $this->xml_encode($entryUri); + + if (empty($entryID)) // Fallback to title and content + $entryID = 'urn:sha1:' . hash('sha1', $entryTitle . $entryContent); + + if (empty($entryTimestamp)) + $entryTimestamp = $this->lastModified; + + if (empty($entryTitle)) { + $entryTitle = str_replace("\n", ' ', strip_tags($entryContent)); + if (strlen($entryTitle) > self::LIMIT_TITLE) { + $wrapPos = strpos(wordwrap($entryTitle, self::LIMIT_TITLE), "\n"); + $entryTitle = substr($entryTitle, 0, $wrapPos) . '...'; + } + } + + if (empty($entryContent)) + $entryContent = $entryTitle; + $entryAuthor = $this->xml_encode($item->getAuthor()); - $entryTitle = $this->xml_encode($item->getTitle()); - $entryUri = $this->xml_encode($item->getURI()); - $entryTimestamp = $this->xml_encode(date(DATE_ATOM, $item->getTimestamp())); - $entryContent = $this->xml_encode($this->sanitizeHtml($item->getContent())); + $entryTitle = $this->xml_encode($entryTitle); + $entryUri = $this->xml_encode($entryUri); + $entryTimestamp = $this->xml_encode(gmdate(DATE_ATOM, $entryTimestamp)); + $entryContent = $this->xml_encode($this->sanitizeHtml($entryContent)); $entryEnclosures = ''; foreach($item->getEnclosures() as $enclosure) { @@ -49,16 +87,28 @@ class AtomFormat extends FormatAbstract{ . PHP_EOL; } + $entryLinkAlternate = ''; + if (!empty($entryUri)) { + $entryLinkAlternate = ''; + } + + if (!empty($entryAuthor)) { + $entryAuthor = '' + . $entryAuthor + . ''; + } + $entries .= << - - {$entryAuthor} - {$entryTitle} - - {$entryUri} + {$entryTimestamp} {$entryTimestamp} + {$entryID} + {$entryLinkAlternate} + {$entryAuthor} {$entryContent} {$entryEnclosures} {$entryCategories} @@ -67,21 +117,24 @@ class AtomFormat extends FormatAbstract{ EOD; } - $feedTimestamp = date(DATE_ATOM, time()); - $charset = $this->getCharset(); + $feedTimestamp = gmdate(DATE_ATOM, $this->lastModified); + $charset = $this->getCharset(); /* Data are prepared, now let's begin the "MAGIE !!!" */ $toReturn = << - + {$title} - http{$https}://{$httpHost}{$httpInfo}/ + {$feedUrl} {$icon} {$icon} {$feedTimestamp} + + {$feedAuthor} + - + {$entries} EOD; diff --git a/formats/HtmlFormat.php b/formats/HtmlFormat.php index 052bedc..ebb6b78 100644 --- a/formats/HtmlFormat.php +++ b/formats/HtmlFormat.php @@ -4,8 +4,21 @@ class HtmlFormat extends FormatAbstract { $extraInfos = $this->getExtraInfos(); $title = htmlspecialchars($extraInfos['name']); $uri = htmlspecialchars($extraInfos['uri']); - $atomquery = str_replace('format=Html', 'format=Atom', htmlentities($_SERVER['QUERY_STRING'])); - $mrssquery = str_replace('format=Html', 'format=Mrss', htmlentities($_SERVER['QUERY_STRING'])); + + // Dynamically build buttons for all formats (except HTML) + $formatFac = new FormatFactory(); + $formatFac->setWorkingDir(PATH_LIB_FORMATS); + + $buttons = ''; + + foreach($formatFac->getFormatNames() as $format) { + if(strcasecmp($format, 'HTML') === 0) { + continue; + } + + $query = str_replace('format=Html', 'format=' . $format, htmlentities($_SERVER['QUERY_STRING'])); + $buttons .= $this->buildButton($format, $query) . PHP_EOL; + } $entries = ''; foreach($this->getItems() as $item) { @@ -82,18 +95,17 @@ EOD; + {$title} - - +

{$title}

- - + {$buttons}
{$entries} @@ -113,4 +125,10 @@ EOD; return parent::display(); } + + private function buildButton($format, $query) { + return << +EOD; + } } diff --git a/formats/JsonFormat.php b/formats/JsonFormat.php index fafe7a5..5d09162 100644 --- a/formats/JsonFormat.php +++ b/formats/JsonFormat.php @@ -16,21 +16,22 @@ class JsonFormat extends FormatAbstract { 'content', 'enclosures', 'categories', + 'uid', ); public function stringify(){ - $urlScheme = (isset($_SERVER['HTTPS']) && $_SERVER['HTTPS'] == 'on') ? 'https://' : 'http://'; - $urlHost = (isset($_SERVER['HTTP_HOST'])) ? $_SERVER['HTTP_HOST'] : ''; - $urlPath = (isset($_SERVER['PATH_INFO'])) ? $_SERVER['PATH_INFO'] : ''; - $urlRequest = (isset($_SERVER['REQUEST_URI'])) ? $_SERVER['REQUEST_URI'] : ''; + $urlPrefix = (isset($_SERVER['HTTPS']) && $_SERVER['HTTPS'] == 'on') ? 'https://' : 'http://'; + $urlHost = (isset($_SERVER['HTTP_HOST'])) ? $_SERVER['HTTP_HOST'] : ''; + $urlPath = (isset($_SERVER['PATH_INFO'])) ? $_SERVER['PATH_INFO'] : ''; + $urlRequest = (isset($_SERVER['REQUEST_URI'])) ? $_SERVER['REQUEST_URI'] : ''; $extraInfos = $this->getExtraInfos(); $data = array( - 'version' => 'https://jsonfeed.org/version/1', - 'title' => (!empty($extraInfos['name'])) ? $extraInfos['name'] : $urlHost, - 'home_page_url' => (!empty($extraInfos['uri'])) ? $extraInfos['uri'] : REPOSITORY, - 'feed_url' => $urlScheme . $urlHost . $urlRequest + 'version' => 'https://jsonfeed.org/version/1', + 'title' => (!empty($extraInfos['name'])) ? $extraInfos['name'] : $urlHost, + 'home_page_url' => (!empty($extraInfos['uri'])) ? $extraInfos['uri'] : REPOSITORY, + 'feed_url' => $urlPrefix . $urlHost . $urlRequest ); if (!empty($extraInfos['icon'])) { @@ -42,20 +43,24 @@ class JsonFormat extends FormatAbstract { foreach ($this->getItems() as $item) { $entry = array(); - $entryAuthor = $item->getAuthor(); - $entryTitle = $item->getTitle(); - $entryUri = $item->getURI(); - $entryTimestamp = $item->getTimestamp(); - $entryContent = $this->sanitizeHtml($item->getContent()); - $entryEnclosures = $item->getEnclosures(); - $entryCategories = $item->getCategories(); + $entryAuthor = $item->getAuthor(); + $entryTitle = $item->getTitle(); + $entryUri = $item->getURI(); + $entryTimestamp = $item->getTimestamp(); + $entryContent = $this->sanitizeHtml($item->getContent()); + $entryEnclosures = $item->getEnclosures(); + $entryCategories = $item->getCategories(); $vendorFields = $item->toArray(); foreach (self::VENDOR_EXCLUDES as $key) { unset($vendorFields[$key]); } - $entry['id'] = $entryUri; + $entry['id'] = $item->getUid(); + + if (empty($entry['id'])) { + $entry['id'] = $entryUri; + } if (!empty($entryTitle)) { $entry['title'] = $entryTitle; @@ -82,8 +87,8 @@ class JsonFormat extends FormatAbstract { $entry['attachments'] = array(); foreach ($entryEnclosures as $enclosure) { $entry['attachments'][] = array( - 'url' => $enclosure, - 'mime_type' => getMimeType($enclosure) + 'url' => $enclosure, + 'mime_type' => getMimeType($enclosure) ); } } diff --git a/formats/MrssFormat.php b/formats/MrssFormat.php index 34b9a92..836a361 100644 --- a/formats/MrssFormat.php +++ b/formats/MrssFormat.php @@ -1,18 +1,45 @@ xml_encode($_SERVER['REQUEST_URI']) : ''; + $feedUrl = $this->xml_encode($urlPrefix . $urlHost . $urlRequest); $extraInfos = $this->getExtraInfos(); $title = $this->xml_encode($extraInfos['name']); + $icon = $extraInfos['icon']; if(!empty($extraInfos['uri'])) { $uri = $this->xml_encode($extraInfos['uri']); @@ -20,34 +47,48 @@ class MrssFormat extends FormatAbstract { $uri = REPOSITORY; } - $uriparts = parse_url($uri); - $icon = $this->xml_encode($uriparts['scheme'] . '://' . $uriparts['host'] . '/favicon.ico'); - $items = ''; foreach($this->getItems() as $item) { - $itemAuthor = $this->xml_encode($item->getAuthor()); + $itemTimestamp = $item->getTimestamp(); $itemTitle = $this->xml_encode($item->getTitle()); $itemUri = $this->xml_encode($item->getURI()); - $itemTimestamp = $this->xml_encode(date(DATE_RFC2822, $item->getTimestamp())); $itemContent = $this->xml_encode($this->sanitizeHtml($item->getContent())); + $entryID = $item->getUid(); + $isPermaLink = 'false'; + + if (empty($entryID) && !empty($itemUri)) { // Fallback to provided URI + $entryID = $itemUri; + $isPermaLink = 'true'; + } + + if (empty($entryID)) // Fallback to title and content + $entryID = hash('sha1', $itemTitle . $itemContent); + + $entryTitle = ''; + if (!empty($itemTitle)) + $entryTitle = '' . $itemTitle . ''; + + $entryLink = ''; + if (!empty($itemUri)) + $entryLink = '' . $itemUri . ''; + + $entryPublished = ''; + if (!empty($itemTimestamp)) { + $entryPublished = '' + . $this->xml_encode(gmdate(DATE_RFC2822, $itemTimestamp)) + . ''; + } + + $entryDescription = ''; + if (!empty($itemContent)) + $entryDescription = '' . $itemContent . ''; - $entryEnclosuresWarning = ''; $entryEnclosures = ''; - if(!empty($item->getEnclosures())) { - $entryEnclosures .= ''; - - if(count($item->getEnclosures()) > 1) { - $entryEnclosures .= PHP_EOL; - $entryEnclosuresWarning = '<br>Warning: -Some media files might not be shown to you. Consider using the ATOM format instead!'; - foreach($item->getEnclosures() as $enclosure) { - $entryEnclosures .= '' - . PHP_EOL; - } - } + foreach($item->getEnclosures() as $enclosure) { + $entryEnclosures .= '' + . PHP_EOL; } $entryCategories = ''; @@ -60,12 +101,11 @@ Some media files might not be shown to you. Consider using the ATOM format inste $items .= << - {$itemTitle} - {$itemUri} - {$itemUri} - {$itemTimestamp} - {$itemContent}{$entryEnclosuresWarning} - {$itemAuthor} + {$entryTitle} + {$entryLink} + {$entryID} + {$entryPublished} + {$entryDescription} {$entryEnclosures} {$entryCategories} @@ -75,22 +115,28 @@ EOD; $charset = $this->getCharset(); - /* xml attributes need to have certain characters escaped to be w3c compliant */ - $imageTitle = htmlspecialchars($title, ENT_COMPAT); + $feedImage = ''; + if (!empty($icon) && in_array(substr($icon, -4), self::ALLOWED_IMAGE_EXT)) { + $feedImage .= << + {$icon} + {$title} + {$uri} + +EOD; + } + /* Data are prepared, now let's begin the "MAGIE !!!" */ $toReturn = << - + {$title} - http{$https}://{$httpHost}{$httpInfo}/ + {$uri} {$title} - - - + {$feedImage} + + {$items} diff --git a/index.php b/index.php index a95302a..666b9e4 100644 --- a/index.php +++ b/index.php @@ -6,8 +6,6 @@ Configuration::loadConfiguration(); Authentication::showPromptIfNeeded(); -date_default_timezone_set('UTC'); - /* Move the CLI arguments to the $_GET array, in order to be able to use rss-bridge from the command line @@ -29,309 +27,17 @@ define('USER_AGENT', ini_set('user_agent', USER_AGENT); -// default whitelist -$whitelist_default = array( - 'BandcampBridge', - 'CryptomeBridge', - 'DansTonChatBridge', - 'DuckDuckGoBridge', - 'FacebookBridge', - 'FlickrBridge', - 'GooglePlusPostBridge', - 'GoogleSearchBridge', - 'IdenticaBridge', - 'InstagramBridge', - 'OpenClassroomsBridge', - 'PinterestBridge', - 'ScmbBridge', - 'TwitterBridge', - 'WikipediaBridge', - 'YoutubeBridge'); - try { - Bridge::setWhitelist($whitelist_default); - - $showInactive = filter_input(INPUT_GET, 'show_inactive', FILTER_VALIDATE_BOOLEAN); - $action = array_key_exists('action', $params) ? $params['action'] : null; - $bridge = array_key_exists('bridge', $params) ? $params['bridge'] : null; - - // Return list of bridges as JSON formatted text - if($action === 'list') { - - $list = new StdClass(); - $list->bridges = array(); - $list->total = 0; - - foreach(Bridge::getBridgeNames() as $bridgeName) { - - $bridge = Bridge::create($bridgeName); - - if($bridge === false) { // Broken bridge, show as inactive - - $list->bridges[$bridgeName] = array( - 'status' => 'inactive' - ); - - continue; - - } - - $status = Bridge::isWhitelisted($bridgeName) ? 'active' : 'inactive'; - - $list->bridges[$bridgeName] = array( - 'status' => $status, - 'uri' => $bridge->getURI(), - 'name' => $bridge->getName(), - 'icon' => $bridge->getIcon(), - 'parameters' => $bridge->getParameters(), - 'maintainer' => $bridge->getMaintainer(), - 'description' => $bridge->getDescription() - ); - - } - - $list->total = count($list->bridges); - - header('Content-Type: application/json'); - echo json_encode($list, JSON_PRETTY_PRINT); - - } elseif($action === 'detect') { - - $targetURL = $params['url'] - or returnClientError('You must specify a url!'); - - $format = $params['format'] - or returnClientError('You must specify a format!'); - - foreach(Bridge::getBridgeNames() as $bridgeName) { - - if(!Bridge::isWhitelisted($bridgeName)) { - continue; - } - - $bridge = Bridge::create($bridgeName); - - if($bridge === false) { - continue; - } - - $bridgeParams = $bridge->detectParameters($targetURL); - - if(is_null($bridgeParams)) { - continue; - } - - $bridgeParams['bridge'] = $bridgeName; - $bridgeParams['format'] = $format; - - header('Location: ?action=display&' . http_build_query($bridgeParams), true, 301); - die(); - - } - - returnClientError('No bridge found for given URL: ' . $targetURL); - - } elseif($action === 'display' && !empty($bridge)) { - - $format = $params['format'] - or returnClientError('You must specify a format!'); - - // DEPRECATED: 'nameFormat' scheme is replaced by 'name' in format parameter values - // this is to keep compatibility until futher complete removal - if(($pos = strpos($format, 'Format')) === (strlen($format) - strlen('Format'))) { - $format = substr($format, 0, $pos); - } - - // whitelist control - if(!Bridge::isWhitelisted($bridge)) { - throw new \Exception('This bridge is not whitelisted', 401); - die; - } - - // Data retrieval - $bridge = Bridge::create($bridge); - - $noproxy = array_key_exists('_noproxy', $params) && filter_var($params['_noproxy'], FILTER_VALIDATE_BOOLEAN); - if(defined('PROXY_URL') && PROXY_BYBRIDGE && $noproxy) { - define('NOPROXY', true); - } - - // Cache timeout - $cache_timeout = -1; - if(array_key_exists('_cache_timeout', $params)) { - - if(!CUSTOM_CACHE_TIMEOUT) { - unset($params['_cache_timeout']); - $uri = parse_url($_SERVER['REQUEST_URI'], PHP_URL_PATH) . '?' . http_build_query($params); - header('Location: ' . $uri, true, 301); - die(); - } - - $cache_timeout = filter_var($params['_cache_timeout'], FILTER_VALIDATE_INT); - - } else { - $cache_timeout = $bridge->getCacheTimeout(); - } - - // Remove parameters that don't concern bridges - $bridge_params = array_diff_key( - $params, - array_fill_keys( - array( - 'action', - 'bridge', - 'format', - '_noproxy', - '_cache_timeout', - '_error_time' - ), '') - ); - - // Remove parameters that don't concern caches - $cache_params = array_diff_key( - $params, - array_fill_keys( - array( - 'action', - 'format', - '_noproxy', - '_cache_timeout', - '_error_time' - ), '') - ); - - // Initialize cache - $cache = Cache::create('FileCache'); - $cache->setPath(PATH_CACHE); - $cache->purgeCache(86400); // 24 hours - $cache->setParameters($cache_params); - - $items = array(); - $infos = array(); - $mtime = $cache->getTime(); - - if($mtime !== false - && (time() - $cache_timeout < $mtime) - && !Debug::isEnabled()) { // Load cached data - - // Send "Not Modified" response if client supports it - // Implementation based on https://stackoverflow.com/a/10847262 - if(isset($_SERVER['HTTP_IF_MODIFIED_SINCE'])) { - $stime = strtotime($_SERVER['HTTP_IF_MODIFIED_SINCE']); - - if($mtime <= $stime) { // Cached data is older or same - header('Last-Modified: ' . gmdate('D, d M Y H:i:s ', $mtime) . 'GMT', true, 304); - die(); - } - } - - $cached = $cache->loadData(); - - if(isset($cached['items']) && isset($cached['extraInfos'])) { - foreach($cached['items'] as $item) { - $items[] = new \FeedItem($item); - } - - $infos = $cached['extraInfos']; - } - - } else { // Collect new data - - try { - $bridge->setDatas($bridge_params); - $bridge->collectData(); - - $items = $bridge->getItems(); - - // Transform "legacy" items to FeedItems if necessary. - // Remove this code when support for "legacy" items ends! - if(isset($items[0]) && is_array($items[0])) { - $feedItems = array(); - - foreach($items as $item) { - $feedItems[] = new \FeedItem($item); - } - - $items = $feedItems; - } - - $infos = array( - 'name' => $bridge->getName(), - 'uri' => $bridge->getURI(), - 'icon' => $bridge->getIcon() - ); - } catch(Error $e) { - error_log($e); - - $item = new \FeedItem(); - - // Create "new" error message every 24 hours - $params['_error_time'] = urlencode((int)(time() / 86400)); - - // Error 0 is a special case (i.e. "trying to get property of non-object") - if($e->getCode() === 0) { - $item->setTitle('Bridge encountered an unexpected situation! (' . $params['_error_time'] . ')'); - } else { - $item->setTitle('Bridge returned error ' . $e->getCode() . '! (' . $params['_error_time'] . ')'); - } - - $item->setURI( - (isset($_SERVER['REQUEST_URI']) ? parse_url($_SERVER['REQUEST_URI'], PHP_URL_PATH) : '') - . '?' - . http_build_query($params) - ); - - $item->setTimestamp(time()); - $item->setContent(buildBridgeException($e, $bridge)); - - $items[] = $item; - } catch(Exception $e) { - error_log($e); - - $item = new \FeedItem(); - - // Create "new" error message every 24 hours - $params['_error_time'] = urlencode((int)(time() / 86400)); - - $item->setURI( - (isset($_SERVER['REQUEST_URI']) ? parse_url($_SERVER['REQUEST_URI'], PHP_URL_PATH) : '') - . '?' - . http_build_query($params) - ); - - $item->setTitle('Bridge returned error ' . $e->getCode() . '! (' . $params['_error_time'] . ')'); - $item->setTimestamp(time()); - $item->setContent(buildBridgeException($e, $bridge)); - - $items[] = $item; - } - - // Store data in cache - $cache->saveData(array( - 'items' => array_map(function($i){ return $i->toArray(); }, $items), - 'extraInfos' => $infos - )); - - } + $actionFac = new \ActionFactory(); + $actionFac->setWorkingDir(PATH_LIB_ACTIONS); - // Data transformation - try { - $format = Format::create($format); - $format->setItems($items); - $format->setExtraInfos($infos); - $format->setLastModified($cache->getTime()); - $format->display(); - } catch(Error $e) { - error_log($e); - header('Content-Type: text/html', true, $e->getCode()); - die(buildTransformException($e, $bridge)); - } catch(Exception $e) { - error_log($e); - header('Content-Type: text/html', true, $e->getCode()); - die(buildTransformException($e, $bridge)); - } + if(array_key_exists('action', $params)) { + $action = $actionFac->create($params['action']); + $action->setUserData($params); + $action->execute(); } else { + $showInactive = filter_input(INPUT_GET, 'show_inactive', FILTER_VALIDATE_BOOLEAN); echo BridgeList::create($showInactive); } } catch(\Exception $e) { diff --git a/lib/ActionAbstract.php b/lib/ActionAbstract.php new file mode 100644 index 0000000..b925d60 --- /dev/null +++ b/lib/ActionAbstract.php @@ -0,0 +1,33 @@ +userData = $userData; + } +} diff --git a/lib/ActionFactory.php b/lib/ActionFactory.php new file mode 100644 index 0000000..8146e54 --- /dev/null +++ b/lib/ActionFactory.php @@ -0,0 +1,65 @@ +buildFilePath($name); + + if(!file_exists($filePath)) { + throw new \Exception('File ' . $filePath . ' does not exist!'); + } + + require_once $filePath; + + $class = $this->buildClassName($name); + + if((new \ReflectionClass($class))->isInstantiable()) { + return new $class(); + } + + return false; + } + + /** + * Build class name from action name + * + * The class name consists of the action name with prefix "Action". The first + * character of the class name must be uppercase. + * + * Example: 'display' => 'DisplayAction' + * + * @param string $name The action name. + * @return string The class name. + */ + protected function buildClassName($name) { + return ucfirst(strtolower($name)) . 'Action'; + } + + /** + * Build file path to the action class. + * + * @param string $name The action name. + * @return string Path to the action class. + */ + protected function buildFilePath($name) { + return $this->getWorkingDir() . $this->buildClassName($name) . '.php'; + } +} diff --git a/lib/ActionInterface.php b/lib/ActionInterface.php new file mode 100644 index 0000000..c38d057 --- /dev/null +++ b/lib/ActionInterface.php @@ -0,0 +1,34 @@ +isInstantiable()) { - return new $name(); - } - - return false; - } - - /** - * Sets the working directory. - * - * @param string $dir Path to the directory containing bridges. - * @throws \LogicException if the provided path is not a valid string. - * @throws \Exception if the provided path does not exist. - * @throws \InvalidArgumentException if $dir is not a directory. - * @return void - */ - public static function setWorkingDir($dir){ - self::$workingDir = null; - - if(!is_string($dir)) { - throw new \InvalidArgumentException('Working directory is not a valid string!'); - } - - if(!file_exists($dir)) { - throw new \Exception('Working directory does not exist!'); - } - - if(!is_dir($dir)) { - throw new \InvalidArgumentException('Working directory is not a directory!'); - } - - self::$workingDir = realpath($dir) . '/'; - } - - /** - * Returns the working directory. - * The working directory must be specified with {@see Bridge::setWorkingDir()}! - * - * @throws \LogicException if the working directory is not set. - * @return string The current working directory. - */ - public static function getWorkingDir(){ - if(is_null(self::$workingDir)) { - throw new \LogicException('Working directory is not set!'); - } - - return self::$workingDir; - } - - /** - * Returns true if the provided name is a valid bridge name. - * - * A valid bridge name starts with a capital letter ([A-Z]), followed by - * zero or more alphanumeric characters or hyphen ([A-Za-z0-9-]). - * - * @param string $name The bridge name. - * @return bool true if the name is a valid bridge name, false otherwise. - */ - public static function isBridgeName($name){ - return is_string($name) && preg_match('/^[A-Z][a-zA-Z0-9-]*$/', $name) === 1; - } - - /** - * Returns the list of bridge names from the working directory. - * - * The list is cached internally to allow for successive calls. - * - * @return array List of bridge names - */ - public static function getBridgeNames(){ - - static $bridgeNames = array(); // Initialized on first call - - if(empty($bridgeNames)) { - $files = scandir(self::getWorkingDir()); - - if($files !== false) { - foreach($files as $file) { - if(preg_match('/^([^.]+)Bridge\.php$/U', $file, $out)) { - $bridgeNames[] = $out[1]; - } - } - } - } - - return $bridgeNames; - } - - /** - * Checks if a bridge is whitelisted. - * - * @param string $name Name of the bridge. - * @return bool True if the bridge is whitelisted. - */ - public static function isWhitelisted($name){ - return in_array(self::sanitizeBridgeName($name), self::getWhitelist()); - } - - /** - * Returns the whitelist. - * - * On first call this function reads the whitelist from {@see WHITELIST}. - * * Each line in the file specifies one bridge on the whitelist. - * * An empty file disables all bridges. - * * If the file only only contains `*`, all bridges are whitelisted. - * - * Use {@see Bridge::setWhitelist()} to specify a default whitelist **before** - * calling this function! The list is cached internally to allow for - * successive calls. If {@see Bridge::setWhitelist()} gets called after this - * function, the whitelist is **not** updated again! - * - * @return array Array of whitelisted bridges - */ - public static function getWhitelist() { - - static $firstCall = true; // Initialized on first call - - if($firstCall) { - - // Create initial whitelist or load from disk - if (!file_exists(WHITELIST) && !empty(self::$whitelist)) { - file_put_contents(WHITELIST, implode("\n", self::$whitelist)); - } else { - - $contents = trim(file_get_contents(WHITELIST)); - - if($contents === '*') { // Whitelist all bridges - self::$whitelist = self::getBridgeNames(); - } else { - self::$whitelist = array_map('self::sanitizeBridgeName', explode("\n", $contents)); - } - - } - - } - - return self::$whitelist; - - } - - /** - * Sets the (default) whitelist. - * - * If this function is called **before** {@see Bridge::getWhitelist()}, the - * provided whitelist will be replaced by a custom whitelist specified in - * {@see WHITELIST} (if it exists). - * - * If this function is called **after** {@see Bridge::getWhitelist()}, the - * provided whitelist is taken as is (not updated by the custom whitelist - * again). - * - * @param array $default The whitelist as array of bridge names. - * @return void - */ - public static function setWhitelist($default = array()) { - self::$whitelist = array_map('self::sanitizeBridgeName', $default); - } - - /** - * Returns the sanitized bridge name. - * - * The bridge name can be specified in various ways: - * * The PHP file name (i.e. `GitHubIssueBridge.php`) - * * The PHP file name without file extension (i.e. `GitHubIssueBridge`) - * * The bridge name (i.e. `GitHubIssue`) - * - * Casing is ignored (i.e. `GITHUBISSUE` and `githubissue` are the same). - * - * A bridge file matching the given bridge name must exist in the working - * directory! - * - * @param string $name The bridge name - * @return string|null The sanitized bridge name if the provided name is - * valid, null otherwise. - */ - protected static function sanitizeBridgeName($name) { - - if(is_string($name)) { - - // Trim trailing '.php' if exists - if(preg_match('/(.+)(?:\.php)/', $name, $matches)) { - $name = $matches[1]; - } - - // Trim trailing 'Bridge' if exists - if(preg_match('/(.+)(?:Bridge)/i', $name, $matches)) { - $name = $matches[1]; - } - - // The name is valid if a corresponding bridge file is found on disk - if(in_array(strtolower($name), array_map('strtolower', self::getBridgeNames()))) { - $index = array_search(strtolower($name), array_map('strtolower', self::getBridgeNames())); - return self::getBridgeNames()[$index]; - } - - Debug::log('Invalid bridge name specified: "' . $name . '"!'); - - } - - return null; // Bad parameter - - } -} diff --git a/lib/BridgeAbstract.php b/lib/BridgeAbstract.php index 13215a4..b4eb9ff 100644 --- a/lib/BridgeAbstract.php +++ b/lib/BridgeAbstract.php @@ -194,6 +194,11 @@ abstract class BridgeAbstract implements BridgeInterface { */ public function setDatas(array $inputs){ + if(isset($inputs['context'])) { // Context hinting (optional) + $this->queriedContext = $inputs['context']; + unset($inputs['context']); + } + if(empty(static::PARAMETERS)) { if(!empty($inputs)) { @@ -218,8 +223,11 @@ abstract class BridgeAbstract implements BridgeInterface { ); } - // Guess the paramter context from input data - $this->queriedContext = $validator->getQueriedContext($inputs, static::PARAMETERS); + // Guess the context from input data + if(empty($this->queriedContext)) { + $this->queriedContext = $validator->getQueriedContext($inputs, static::PARAMETERS); + } + if(is_null($this->queriedContext)) { returnClientError('Required parameter(s) missing'); } elseif($this->queriedContext === false) { diff --git a/lib/BridgeCard.php b/lib/BridgeCard.php index a3493b7..c6f3822 100644 --- a/lib/BridgeCard.php +++ b/lib/BridgeCard.php @@ -48,13 +48,19 @@ final class BridgeCard { * @param bool $isHttps If disabled, adds a warning to the form * @return string The form header */ - private static function getFormHeader($bridgeName, $isHttps = false) { + private static function getFormHeader($bridgeName, $isHttps = false, $parameterName = '') { $form = << EOD; + if(!empty($parameterName)) { + $form .= << +EOD; + } + if(!$isHttps) { $form .= '
Warning : This bridge is not fetching its content through a secure connection
'; @@ -80,7 +86,7 @@ This bridge is not fetching its content through a secure connection
'; $isHttps = false, $parameterName = '', $parameters = array()) { - $form = self::getFormHeader($bridgeName, $isHttps); + $form = self::getFormHeader($bridgeName, $isHttps, $parameterName); if(count($parameters) > 0) { @@ -207,6 +213,11 @@ This bridge is not fetching its content through a secure connection
'; * @return string The list input field */ private static function getListInput($entry, $id, $name) { + if(isset($entry['required']) && $entry['required'] === true) { + Debug::log('The "required" attribute is not supported for lists.'); + unset($entry['required']); + } + $list = ' EOD; diff --git a/lib/Cache.php b/lib/Cache.php deleted file mode 100644 index a0d2ac7..0000000 --- a/lib/Cache.php +++ /dev/null @@ -1,140 +0,0 @@ -isInstantiable()) { - return new $name(); - } - - return false; - } - - /** - * Sets the working directory. - * - * @param string $dir Path to a directory containing cache classes - * @throws \InvalidArgumentException if $dir is not a string. - * @throws \Exception if the working directory doesn't exist. - * @throws \InvalidArgumentException if $dir is not a directory. - * @return void - */ - public static function setWorkingDir($dir){ - self::$workingDir = null; - - if(!is_string($dir)) { - throw new \InvalidArgumentException('Working directory is not a valid string!'); - } - - if(!file_exists($dir)) { - throw new \Exception('Working directory does not exist!'); - } - - if(!is_dir($dir)) { - throw new \InvalidArgumentException('Working directory is not a directory!'); - } - - self::$workingDir = realpath($dir) . '/'; - } - - /** - * Returns the working directory. - * The working directory must be set with {@see Cache::setWorkingDir()}! - * - * @throws \LogicException if the working directory is not set. - * @return string The current working directory. - */ - public static function getWorkingDir(){ - if(is_null(self::$workingDir)) { - throw new \LogicException('Working directory is not set!'); - } - - return self::$workingDir; - } - - /** - * Returns true if the provided name is a valid cache name. - * - * A valid cache name starts with a capital letter ([A-Z]), followed by - * zero or more alphanumeric characters or hyphen ([A-Za-z0-9-]). - * - * @param string $name The cache name. - * @return bool true if the name is a valid cache name, false otherwise. - */ - public static function isCacheName($name){ - return is_string($name) && preg_match('/^[A-Z][a-zA-Z0-9-]*$/', $name) === 1; - } -} diff --git a/lib/CacheFactory.php b/lib/CacheFactory.php new file mode 100644 index 0000000..9ce5c19 --- /dev/null +++ b/lib/CacheFactory.php @@ -0,0 +1,149 @@ +sanitizeCacheName($name) . 'Cache'; + + if(!$this->isCacheName($name)) { + throw new \InvalidArgumentException('Cache name invalid!'); + } + + $filePath = $this->getWorkingDir() . $name . '.php'; + + if(!file_exists($filePath)) { + throw new \Exception('Cache file ' . $filePath . ' does not exist!'); + } + + require_once $filePath; + + if((new \ReflectionClass($name))->isInstantiable()) { + return new $name(); + } + + return false; + } + + /** + * Returns true if the provided name is a valid cache name. + * + * A valid cache name starts with a capital letter ([A-Z]), followed by + * zero or more alphanumeric characters or hyphen ([A-Za-z0-9-]). + * + * @param string $name The cache name. + * @return bool true if the name is a valid cache name, false otherwise. + */ + public function isCacheName($name){ + return is_string($name) && preg_match('/^[A-Z][a-zA-Z0-9-]*$/', $name) === 1; + } + + /** + * Returns a list of cache names from the working directory. + * + * The list is cached internally to allow for successive calls. + * + * @return array List of cache names + */ + public function getCacheNames(){ + + static $cacheNames = array(); // Initialized on first call + + if(empty($cacheNames)) { + $files = scandir($this->getWorkingDir()); + + if($files !== false) { + foreach($files as $file) { + if(preg_match('/^([^.]+)Cache\.php$/U', $file, $out)) { + $cacheNames[] = $out[1]; + } + } + } + } + + return $cacheNames; + } + + /** + * Returns the sanitized cache name. + * + * The cache name can be specified in various ways: + * * The PHP file name (i.e. `FileCache.php`) + * * The PHP file name without file extension (i.e. `FileCache`) + * * The cache name (i.e. `file`) + * + * Casing is ignored (i.e. `FILE` and `fIlE` are the same). + * + * A cache file matching the given cache name must exist in the working + * directory! + * + * @param string $name The cache name + * @return string|null The sanitized cache name if the provided name is + * valid, null otherwise. + */ + protected function sanitizeCacheName($name) { + + if(is_string($name)) { + + // Trim trailing '.php' if exists + if(preg_match('/(.+)(?:\.php)/', $name, $matches)) { + $name = $matches[1]; + } + + // Trim trailing 'Cache' if exists + if(preg_match('/(.+)(?:Cache)$/i', $name, $matches)) { + $name = $matches[1]; + } + + // The name is valid if a corresponding file is found on disk + if(in_array(strtolower($name), array_map('strtolower', $this->getCacheNames()))) { + $index = array_search(strtolower($name), array_map('strtolower', $this->getCacheNames())); + return $this->getCacheNames()[$index]; + } + + Debug::log('Invalid cache name specified: "' . $name . '"!'); + + } + + return null; // Bad parameter + + } +} diff --git a/lib/CacheInterface.php b/lib/CacheInterface.php index a74fc0d..091c5f0 100644 --- a/lib/CacheInterface.php +++ b/lib/CacheInterface.php @@ -13,38 +13,54 @@ /** * The cache interface - * - * @todo Add missing function to the interface - * @todo Explain parameters and return values in more detail - * @todo Return self more often (to allow call chaining) */ interface CacheInterface { + /** + * Set scope of the current cache + * + * If $scope is an empty string, the cache is set to a global context. + * + * @param string $scope The scope the data is related to + */ + public function setScope($scope); + + /** + * Set key to assign the current data + * + * Since $key can be anything, the cache implementation must ensure to + * assign the related data reliably; most commonly by serializing and + * hashing the key in an appropriate way. + * + * @param array $key The key the data is related to + */ + public function setKey($key); + /** * Loads data from cache * - * @return mixed The cache data + * @return mixed The cached data or null */ public function loadData(); /** * Stores data to the cache * - * @param mixed $datas The data to store + * @param mixed $data The data to store * @return self The cache object */ - public function saveData($datas); + public function saveData($data); /** - * Returns the timestamp for the curent cache file + * Returns the timestamp for the curent cache data * - * @return int Timestamp + * @return int Timestamp or null */ public function getTime(); /** - * Removes any data that is older than the specified duration from cache + * Removes any data that is older than the specified age from cache * - * @param int $duration The cache duration in seconds + * @param int $seconds The cache age in seconds */ - public function purgeCache($duration); + public function purgeCache($seconds); } diff --git a/lib/Configuration.php b/lib/Configuration.php index 64d9dde..fc575d6 100644 --- a/lib/Configuration.php +++ b/lib/Configuration.php @@ -28,7 +28,7 @@ final class Configuration { * * @todo Replace this property by a constant. */ - public static $VERSION = '2019-01-13'; + public static $VERSION = '2019-09-12'; /** * Holds the configuration data. @@ -80,35 +80,31 @@ final class Configuration { // Check PHP version if(version_compare(PHP_VERSION, '5.6.0') === -1) - die('RSS-Bridge requires at least PHP version 5.6.0!'); + self::reportError('RSS-Bridge requires at least PHP version 5.6.0!'); // extensions check if(!extension_loaded('openssl')) - die('"openssl" extension not loaded. Please check "php.ini"'); + self::reportError('"openssl" extension not loaded. Please check "php.ini"'); if(!extension_loaded('libxml')) - die('"libxml" extension not loaded. Please check "php.ini"'); + self::reportError('"libxml" extension not loaded. Please check "php.ini"'); if(!extension_loaded('mbstring')) - die('"mbstring" extension not loaded. Please check "php.ini"'); + self::reportError('"mbstring" extension not loaded. Please check "php.ini"'); if(!extension_loaded('simplexml')) - die('"simplexml" extension not loaded. Please check "php.ini"'); + self::reportError('"simplexml" extension not loaded. Please check "php.ini"'); // Allow RSS-Bridge to run without curl module in CLI mode without root certificates if(!extension_loaded('curl') && !(php_sapi_name() === 'cli' && empty(ini_get('curl.cainfo')))) - die('"curl" extension not loaded. Please check "php.ini"'); + self::reportError('"curl" extension not loaded. Please check "php.ini"'); if(!extension_loaded('json')) - die('"json" extension not loaded. Please check "php.ini"'); + self::reportError('"json" extension not loaded. Please check "php.ini"'); // Check cache folder permissions (write permissions required) if(!is_writable(PATH_CACHE)) - die('RSS-Bridge does not have write permissions for ' . PATH_CACHE . '!'); - - // Check whitelist file permissions - if(!file_exists(WHITELIST) && !is_writable(dirname(WHITELIST))) - die('RSS-Bridge does not have write permissions for ' . WHITELIST . '!'); + self::reportError('RSS-Bridge does not have write permissions for ' . PATH_CACHE . '!'); } @@ -118,15 +114,13 @@ final class Configuration { * Returns an error message and aborts execution if the configuration is invalid. * * The RSS-Bridge configuration is split into two files: - * - `config.default.ini.php`: The default configuration file that ships with - * every release of RSS-Bridge (do not modify this file!). - * - `config.ini.php`: The local configuration file that can be modified by - * server administrators. - * - * The files must be located at {@see PATH_ROOT} + * - {@see FILE_CONFIG_DEFAULT} The default configuration file that ships + * with every release of RSS-Bridge (do not modify this file!). + * - {@see FILE_CONFIG} The local configuration file that can be modified + * by server administrators. * - * RSS-Bridge will first load `config.default.ini.php` into memory and then - * replace parameters with the contents of `config.ini.php`. That way new + * RSS-Bridge will first load {@see FILE_CONFIG_DEFAULT} into memory and then + * replace parameters with the contents of {@see FILE_CONFIG}. That way new * parameters are automatically initialized with default values and custom * configurations can be reduced to the minimum set of parametes necessary * (only the ones that changed). @@ -140,16 +134,16 @@ final class Configuration { */ public static function loadConfiguration() { - if(!file_exists(PATH_ROOT . 'config.default.ini.php')) - die('The default configuration file "config.default.ini.php" is missing!'); + if(!file_exists(FILE_CONFIG_DEFAULT)) + self::reportError('The default configuration file is missing at ' . FILE_CONFIG_DEFAULT); - Configuration::$config = parse_ini_file(PATH_ROOT . 'config.default.ini.php', true, INI_SCANNER_TYPED); + Configuration::$config = parse_ini_file(FILE_CONFIG_DEFAULT, true, INI_SCANNER_TYPED); if(!Configuration::$config) - die('Error parsing config.default.ini.php'); + self::reportError('Error parsing ' . FILE_CONFIG_DEFAULT); - if(file_exists(PATH_ROOT . 'config.ini.php')) { + if(file_exists(FILE_CONFIG)) { // Replace default configuration with custom settings - foreach(parse_ini_file(PATH_ROOT . 'config.ini.php', true, INI_SCANNER_TYPED) as $header => $section) { + foreach(parse_ini_file(FILE_CONFIG, true, INI_SCANNER_TYPED) as $header => $section) { foreach($section as $key => $value) { // Skip unknown sections and keys if(array_key_exists($header, Configuration::$config) && array_key_exists($key, Configuration::$config[$header])) { @@ -159,8 +153,14 @@ final class Configuration { } } + if(!is_string(self::getConfig('system', 'timezone')) + || !in_array(self::getConfig('system', 'timezone'), timezone_identifiers_list(DateTimeZone::ALL_WITH_BC))) + self::reportConfigurationError('system', 'timezone'); + + date_default_timezone_set(self::getConfig('system', 'timezone')); + if(!is_string(self::getConfig('proxy', 'url'))) - die('Parameter [proxy] => "url" is not a valid string! Please check "config.ini.php"!'); + self::reportConfigurationError('proxy', 'url', 'Is not a valid string'); if(!empty(self::getConfig('proxy', 'url'))) { /** URL of the proxy server */ @@ -168,35 +168,38 @@ final class Configuration { } if(!is_bool(self::getConfig('proxy', 'by_bridge'))) - die('Parameter [proxy] => "by_bridge" is not a valid Boolean! Please check "config.ini.php"!'); + self::reportConfigurationError('proxy', 'by_bridge', 'Is not a valid Boolean'); /** True if proxy usage can be enabled selectively for each bridge */ define('PROXY_BYBRIDGE', self::getConfig('proxy', 'by_bridge')); if(!is_string(self::getConfig('proxy', 'name'))) - die('Parameter [proxy] => "name" is not a valid string! Please check "config.ini.php"!'); + self::reportConfigurationError('proxy', 'name', 'Is not a valid string'); /** Name of the proxy server */ define('PROXY_NAME', self::getConfig('proxy', 'name')); + if(!is_string(self::getConfig('cache', 'type'))) + self::reportConfigurationError('cache', 'type', 'Is not a valid string'); + if(!is_bool(self::getConfig('cache', 'custom_timeout'))) - die('Parameter [cache] => "custom_timeout" is not a valid Boolean! Please check "config.ini.php"!'); + self::reportConfigurationError('cache', 'custom_timeout', 'Is not a valid Boolean'); /** True if the cache timeout can be specified by the user */ define('CUSTOM_CACHE_TIMEOUT', self::getConfig('cache', 'custom_timeout')); if(!is_bool(self::getConfig('authentication', 'enable'))) - die('Parameter [authentication] => "enable" is not a valid Boolean! Please check "config.ini.php"!'); + self::reportConfigurationError('authentication', 'enable', 'Is not a valid Boolean'); if(!is_string(self::getConfig('authentication', 'username'))) - die('Parameter [authentication] => "username" is not a valid string! Please check "config.ini.php"!'); + self::reportConfigurationError('authentication', 'username', 'Is not a valid string'); if(!is_string(self::getConfig('authentication', 'password'))) - die('Parameter [authentication] => "password" is not a valid string! Please check "config.ini.php"!'); + self::reportConfigurationError('authentication', 'password', 'Is not a valid string'); if(!empty(self::getConfig('admin', 'email')) && !filter_var(self::getConfig('admin', 'email'), FILTER_VALIDATE_EMAIL)) - die('Parameter [admin] => "email" is not a valid email address! Please check "config.ini.php"!'); + self::reportConfigurationError('admin', 'email', 'Is not a valid email address'); } @@ -243,4 +246,46 @@ final class Configuration { return Configuration::$VERSION; } + + /** + * Reports an configuration error for the specified section and key to the + * user and ends execution + * + * @param string $section The section name + * @param string $key The configuration key + * @param string $message An optional message to the user + * + * @return void + */ + private static function reportConfigurationError($section, $key, $message = '') { + + $report = "Parameter [{$section}] => \"{$key}\" is invalid!" . PHP_EOL; + + if(file_exists(FILE_CONFIG)) { + $report .= 'Please check your configuration file at ' . FILE_CONFIG . PHP_EOL; + } elseif(!file_exists(FILE_CONFIG_DEFAULT)) { + $report .= 'The default configuration file is missing at ' . FILE_CONFIG_DEFAULT . PHP_EOL; + } else { + $report .= 'The default configuration file is broken.' . PHP_EOL + . 'Restore the original file from ' . REPOSITORY . PHP_EOL; + } + + $report .= $message; + self::reportError($report); + + } + + /** + * Reports an error message to the user and ends execution + * + * @param string $message The error message + * + * @return void + */ + private static function reportError($message) { + + header('Content-Type: text/plain', true, 500); + die('Configuration error' . PHP_EOL . $message); + + } } diff --git a/lib/Exceptions.php b/lib/Exceptions.php index ac452d0..c749780 100644 --- a/lib/Exceptions.php +++ b/lib/Exceptions.php @@ -11,6 +11,15 @@ * @link https://github.com/rss-bridge/rss-bridge */ +/** + * Builds a GitHub search query to find open bugs for the current bridge + */ +function buildGitHubSearchQuery($bridgeName){ + return REPOSITORY + . 'issues?q=' + . urlencode('is:issue is:open ' . $bridgeName); +} + /** * Returns an URL that automatically populates a new issue on GitHub based * on the information provided @@ -83,7 +92,8 @@ function buildBridgeException($e, $bridge){ . '`'; $body_html = nl2br($body); - $link = buildGitHubIssueQuery($title, $body, 'bug report', $bridge->getMaintainer()); + $link = buildGitHubIssueQuery($title, $body, 'Bridge-Broken', $bridge->getMaintainer()); + $searchQuery = buildGitHubSearchQuery($bridge::NAME); $header = buildHeader($e, $bridge); $message = << {$body_html} EOD; - $section = buildSection($e, $bridge, $message, $link); + $section = buildSection($e, $bridge, $message, $link, $searchQuery); return $section; } @@ -119,11 +129,12 @@ function buildTransformException($e, $bridge){ . (isset($_SERVER['QUERY_STRING']) ? $_SERVER['QUERY_STRING'] : '') . '`'; - $link = buildGitHubIssueQuery($title, $body, 'bug report', $bridge->getMaintainer()); + $link = buildGitHubIssueQuery($title, $body, 'Bridge-Broken', $bridge->getMaintainer()); + $searchQuery = buildGitHubSearchQuery($bridge::NAME); $header = buildHeader($e, $bridge); $message = "RSS-Bridge was unable to transform the contents returned by {$bridge->getName()}!"; - $section = buildSection($e, $bridge, $message, $link); + $section = buildSection($e, $bridge, $message, $link, $searchQuery); return buildPage($title, $header, $section); } @@ -154,11 +165,12 @@ EOD; * @param object $bridge The bridge object * @param string $message The message to display * @param string $link The link to include in the anchor + * @param string $searchQuery A GitHub search query for the current bridge * @return string The HTML section * * @todo This function belongs inside a class */ -function buildSection($e, $bridge, $message, $link){ +function buildSection($e, $bridge, $message, $link, $searchQuery){ return <<

{$message}

@@ -166,9 +178,13 @@ function buildSection($e, $bridge, $message, $link){
  • Press Return to check your input parameters
  • Press F5 to retry
    • +
  • Check if this issue was already reported on GitHub (give it a thumbs-up)
  • Open a GitHub Issue if this error persists
+ + +

{$bridge->getMaintainer()}

diff --git a/lib/FactoryAbstract.php b/lib/FactoryAbstract.php new file mode 100644 index 0000000..c91ae2e --- /dev/null +++ b/lib/FactoryAbstract.php @@ -0,0 +1,70 @@ +workingDir = null; + + if(!is_string($dir)) { + throw new \InvalidArgumentException('Working directory must be a string!'); + } + + if(!file_exists($dir)) { + throw new \Exception('Working directory does not exist!'); + } + + if(!is_dir($dir)) { + throw new \InvalidArgumentException($dir . ' is not a directory!'); + } + + $this->workingDir = realpath($dir) . '/'; + } + + /** + * Get the working directory + * + * @return string The working directory. + */ + public function getWorkingDir() { + if(is_null($this->workingDir)) { + throw new \LogicException('Working directory is not set!'); + } + + return $this->workingDir; + } + + /** + * Creates a new instance for the object specified by name. + * + * @param string $name The name of the object to create. + * @return object The object instance + */ + abstract public function create($name); +} diff --git a/lib/FeedExpander.php b/lib/FeedExpander.php index b669351..665620a 100644 --- a/lib/FeedExpander.php +++ b/lib/FeedExpander.php @@ -265,7 +265,7 @@ abstract class FeedExpander extends BridgeAbstract { //When "link" field is present, URL is more reliable than "id" field if (count($feedItem->link) === 1) { - $this->uri = (string)$feedItem->link[0]['href']; + $item['uri'] = (string)$feedItem->link[0]['href']; } else { foreach($feedItem->link as $link) { if(strtolower($link['rel']) === 'alternate') { diff --git a/lib/FeedItem.php b/lib/FeedItem.php index 2812da6..9a43573 100644 --- a/lib/FeedItem.php +++ b/lib/FeedItem.php @@ -55,6 +55,9 @@ class FeedItem { /** @var array List of category names or tags */ protected $categories = array(); + /** @var string Unique ID for the current item */ + protected $uid = null; + /** @var array Associative list of additional parameters */ protected $misc = array(); // Custom parameters @@ -73,7 +76,7 @@ class FeedItem { * $item['uri'] = 'https://www.github.com/rss-bridge/rss-bridge/'; * $item['title'] = 'Title'; * $item['timestamp'] = strtotime('now'); - * $item['autor'] = 'Unknown author'; + * $item['author'] = 'Unknown author'; * $item['content'] = 'Hello World!'; * $item['enclosures'] = array('https://github.com/favicon.ico'); * $item['categories'] = array('php', 'rss-bridge', 'awesome'); @@ -344,7 +347,7 @@ class FeedItem { $enclosure, FILTER_VALIDATE_URL, FILTER_FLAG_PATH_REQUIRED)) { Debug::log('Each enclosure must contain a scheme, host and path!'); - } else { + } elseif(!in_array($enclosure, $this->enclosures)) { $this->enclosures[] = $enclosure; } } @@ -391,6 +394,40 @@ class FeedItem { return $this; } + /** + * Get unique id + * + * Use {@see FeedItem::setUid()} to set the unique id. + * + * @param string The unique id. + */ + public function getUid() { + return $this->uid; + } + + /** + * Set unique id. + * + * Use {@see FeedItem::getUid()} to get the unique id. + * + * @param string $uid A string that uniquely identifies the current item + * @return self + */ + public function setUid($uid) { + $this->uid = null; // Clear previous data + + if(!is_string($uid)) { + Debug::log('Unique id must be a string!'); + } elseif (preg_match('/^[a-f0-9]{40}$/', $uid)) { + // keep id if it already is a SHA-1 hash + $this->uid = $uid; + } else { + $this->uid = sha1($uid); + } + + return $this; + } + /** * Add miscellaneous elements to the item. * @@ -426,6 +463,7 @@ class FeedItem { 'content' => $this->content, 'enclosures' => $this->enclosures, 'categories' => $this->categories, + 'uid' => $this->uid, ), $this->misc ); } @@ -454,6 +492,7 @@ class FeedItem { case 'content': $this->setContent($value); break; case 'enclosures': $this->setEnclosures($value); break; case 'categories': $this->setCategories($value); break; + case 'uid': $this->setUid($value); break; default: $this->addMisc($name, $value); } } @@ -476,6 +515,7 @@ class FeedItem { case 'content': return $this->getContent(); case 'enclosures': return $this->getEnclosures(); case 'categories': return $this->getCategories(); + case 'uid': return $this->getUid(); default: if(array_key_exists($name, $this->misc)) return $this->misc[$name]; diff --git a/lib/Format.php b/lib/Format.php deleted file mode 100644 index 061b1f2..0000000 --- a/lib/Format.php +++ /dev/null @@ -1,166 +0,0 @@ -isInstantiable()) { - return new $name(); - } - - return false; - } - - /** - * Sets the working directory. - * - * @param string $dir Path to a directory containing cache classes - * @throws \InvalidArgumentException if $dir is not a string. - * @throws \Exception if the working directory doesn't exist. - * @throws \InvalidArgumentException if $dir is not a directory. - * @return void - */ - public static function setWorkingDir($dir){ - self::$workingDir = null; - - if(!is_string($dir)) { - throw new \InvalidArgumentException('Dir format must be a string.'); - } - - if(!file_exists($dir)) { - throw new \Exception('Working directory does not exist!'); - } - - if(!is_dir($dir)) { - throw new \InvalidArgumentException('Working directory is not a directory!'); - } - - self::$workingDir = realpath($dir) . '/'; - } - - /** - * Returns the working directory. - * The working directory must be set with {@see Format::setWorkingDir()}! - * - * @throws \LogicException if the working directory is not set. - * @return string The current working directory. - */ - public static function getWorkingDir(){ - if(is_null(self::$workingDir)) { - throw new \LogicException('Working directory is not set!'); - } - - return self::$workingDir; - } - - /** - * Returns true if the provided name is a valid format name. - * - * A valid format name starts with a capital letter ([A-Z]), followed by - * zero or more alphanumeric characters or hyphen ([A-Za-z0-9-]). - * - * @param string $name The format name. - * @return bool true if the name is a valid format name, false otherwise. - */ - public static function isFormatName($name){ - return is_string($name) && preg_match('/^[A-Z][a-zA-Z0-9-]*$/', $name) === 1; - } - - /** - * Returns the list of format names from the working directory. - * - * The list is cached internally to allow for successive calls. - * - * @return array List of format names - */ - public static function getFormatNames(){ - static $formatNames = array(); // Initialized on first call - - if(empty($formatNames)) { - $files = scandir(self::getWorkingDir()); - - if($files !== false) { - foreach($files as $file) { - if(preg_match('/^([^.]+)Format\.php$/U', $file, $out)) { - $formatNames[] = $out[1]; - } - } - } - } - - return $formatNames; - } -} diff --git a/lib/FormatFactory.php b/lib/FormatFactory.php new file mode 100644 index 0000000..28db759 --- /dev/null +++ b/lib/FormatFactory.php @@ -0,0 +1,153 @@ +isFormatName($name)) { + throw new \InvalidArgumentException('Format name invalid!'); + } + + $name = $this->sanitizeFormatName($name) . 'Format'; + $pathFormat = $this->getWorkingDir() . $name . '.php'; + + if(!file_exists($pathFormat)) { + throw new \Exception('Format file ' . $filePath . ' does not exist!'); + } + + require_once $pathFormat; + + if((new \ReflectionClass($name))->isInstantiable()) { + return new $name(); + } + + return false; + } + + /** + * Returns true if the provided name is a valid format name. + * + * A valid format name starts with a capital letter ([A-Z]), followed by + * zero or more alphanumeric characters or hyphen ([A-Za-z0-9-]). + * + * @param string $name The format name. + * @return bool true if the name is a valid format name, false otherwise. + */ + public function isFormatName($name){ + return is_string($name) && preg_match('/^[A-Z][a-zA-Z0-9-]*$/', $name) === 1; + } + + /** + * Returns the list of format names from the working directory. + * + * The list is cached internally to allow for successive calls. + * + * @return array List of format names + */ + public function getFormatNames(){ + static $formatNames = array(); // Initialized on first call + + if(empty($formatNames)) { + $files = scandir($this->getWorkingDir()); + + if($files !== false) { + foreach($files as $file) { + if(preg_match('/^([^.]+)Format\.php$/U', $file, $out)) { + $formatNames[] = $out[1]; + } + } + } + } + + return $formatNames; + } + + /** + * Returns the sanitized format name. + * + * The format name can be specified in various ways: + * * The PHP file name (i.e. `AtomFormat.php`) + * * The PHP file name without file extension (i.e. `AtomFormat`) + * * The format name (i.e. `Atom`) + * + * Casing is ignored (i.e. `ATOM` and `atom` are the same). + * + * A format file matching the given format name must exist in the working + * directory! + * + * @param string $name The format name + * @return string|null The sanitized format name if the provided name is + * valid, null otherwise. + */ + protected function sanitizeFormatName($name) { + + if(is_string($name)) { + + // Trim trailing '.php' if exists + if(preg_match('/(.+)(?:\.php)/', $name, $matches)) { + $name = $matches[1]; + } + + // Trim trailing 'Format' if exists + if(preg_match('/(.+)(?:Format)/i', $name, $matches)) { + $name = $matches[1]; + } + + // Improve performance for correctly written format names + if(in_array($name, $this->getFormatNames())) { + $index = array_search($name, $this->getFormatNames()); + return $this->getFormatNames()[$index]; + } + + // The name is valid if a corresponding format file is found on disk + if(in_array(strtolower($name), array_map('strtolower', $this->getFormatNames()))) { + $index = array_search(strtolower($name), array_map('strtolower', $this->getFormatNames())); + return $this->getFormatNames()[$index]; + } + + Debug::log('Invalid format name: "' . $name . '"!'); + + } + + return null; // Bad parameter + + } +} diff --git a/lib/ParameterValidator.php b/lib/ParameterValidator.php index 91fe7c9..f740888 100644 --- a/lib/ParameterValidator.php +++ b/lib/ParameterValidator.php @@ -195,13 +195,14 @@ class ParameterValidator { foreach($set as $id => $properties) { if(isset($data[$id]) && !empty($data[$id])) { $queriedContexts[$context] = true; - } elseif(isset($properties['required']) - && $properties['required'] === true) { + } elseif (isset($properties['type']) + && ($properties['type'] === 'checkbox' || $properties['type'] === 'list')) { + continue; + } elseif(isset($properties['required']) && $properties['required'] === true) { $queriedContexts[$context] = false; break; } } - } // Abort if one of the globally required parameters is not satisfied @@ -213,6 +214,7 @@ class ParameterValidator { switch(array_sum($queriedContexts)) { case 0: // Found no match, is there a context without parameters? + if(isset($data['context'])) return $data['context']; foreach($queriedContexts as $context => $queried) { if(is_null($queried)) { return $context; diff --git a/lib/contents.php b/lib/contents.php index dc0ca51..8649c0b 100644 --- a/lib/contents.php +++ b/lib/contents.php @@ -45,16 +45,31 @@ function getContents($url, $header = array(), $opts = array()){ Debug::log('Reading contents from "' . $url . '"'); // Initialize cache - $cache = Cache::create('FileCache'); - $cache->setPath(PATH_CACHE . 'server/'); + $cacheFac = new CacheFactory(); + $cacheFac->setWorkingDir(PATH_LIB_CACHES); + $cache = $cacheFac->create(Configuration::getConfig('cache', 'type')); + $cache->setScope('server'); $cache->purgeCache(86400); // 24 hours (forced) $params = [$url]; - $cache->setParameters($params); + $cache->setKey($params); // Use file_get_contents if in CLI mode with no root certificates defined if(php_sapi_name() === 'cli' && empty(ini_get('curl.cainfo'))) { - $data = @file_get_contents($url); + + $httpHeaders = ''; + + foreach ($header as $headerL) { + $httpHeaders .= $headerL . "\r\n"; + } + + $ctx = stream_context_create(array( + 'http' => array( + 'header' => $httpHeaders + ) + )); + + $data = @file_get_contents($url, 0, $ctx); if($data === false) { $errorCode = 500; @@ -207,14 +222,15 @@ EOD * @return string Contents as simplehtmldom object. */ function getSimpleHTMLDOM($url, -$header = array(), -$opts = array(), -$lowercase = true, -$forceTagsClosed = true, -$target_charset = DEFAULT_TARGET_CHARSET, -$stripRN = true, -$defaultBRText = DEFAULT_BR_TEXT, -$defaultSpanText = DEFAULT_SPAN_TEXT){ + $header = array(), + $opts = array(), + $lowercase = true, + $forceTagsClosed = true, + $target_charset = DEFAULT_TARGET_CHARSET, + $stripRN = true, + $defaultBRText = DEFAULT_BR_TEXT, + $defaultSpanText = DEFAULT_SPAN_TEXT){ + $content = getContents($url, $header, $opts); return str_get_html($content, $lowercase, @@ -256,24 +272,27 @@ $defaultSpanText = DEFAULT_SPAN_TEXT){ * @return string Contents as simplehtmldom object. */ function getSimpleHTMLDOMCached($url, -$duration = 86400, -$header = array(), -$opts = array(), -$lowercase = true, -$forceTagsClosed = true, -$target_charset = DEFAULT_TARGET_CHARSET, -$stripRN = true, -$defaultBRText = DEFAULT_BR_TEXT, -$defaultSpanText = DEFAULT_SPAN_TEXT){ + $duration = 86400, + $header = array(), + $opts = array(), + $lowercase = true, + $forceTagsClosed = true, + $target_charset = DEFAULT_TARGET_CHARSET, + $stripRN = true, + $defaultBRText = DEFAULT_BR_TEXT, + $defaultSpanText = DEFAULT_SPAN_TEXT){ + Debug::log('Caching url ' . $url . ', duration ' . $duration); // Initialize cache - $cache = Cache::create('FileCache'); - $cache->setPath(PATH_CACHE . 'pages/'); + $cacheFac = new CacheFactory(); + $cacheFac->setWorkingDir(PATH_LIB_CACHES); + $cache = $cacheFac->create(Configuration::getConfig('cache', 'type')); + $cache->setScope('pages'); $cache->purgeCache(86400); // 24 hours (forced) $params = [$url]; - $cache->setParameters($params); + $cache->setKey($params); // Determine if cached file is within duration $time = $cache->getTime(); @@ -320,8 +339,8 @@ function parseResponseHeader($header) { $header['http_code'] = $line; } else { - list ($key, $value) = explode(': ', $line); - $header[$key] = $value; + list ($key, $value) = explode(':', $line); + $header[$key] = trim($value); } diff --git a/lib/html.php b/lib/html.php index e49ca7a..13db97a 100644 --- a/lib/html.php +++ b/lib/html.php @@ -26,23 +26,13 @@ * already removes some of the tags (search for `remove_noise` in simple_html_dom.php). */ function sanitize($html, -$tags_to_remove = array('script', 'iframe', 'input', 'form'), -$attributes_to_keep = array('title', 'href', 'src'), -$text_to_keep = array()){ + $tags_to_remove = array('script', 'iframe', 'input', 'form'), + $attributes_to_keep = array('title', 'href', 'src'), + $text_to_keep = array()){ + $htmlContent = str_get_html($html); - /* - * Notice: simple_html_dom currently doesn't support "->find(*)", which is a - * known issue: https://sourceforge.net/p/simplehtmldom/bugs/157/ - * - * A solution to this is to find all nodes WITHOUT a specific attribute. If - * the attribute is very unlikely to appear in the DOM, this is essentially - * returning all nodes. - * - * "*[!b38fd2b1fe7f4747d6b1c1254ccd055e]" is doing exactly that. The attrib - * "b38fd2b1fe7f4747d6b1c1254ccd055e" is very unlikely to appear in any DOM. - */ - foreach($htmlContent->find('*[!b38fd2b1fe7f4747d6b1c1254ccd055e]') as $element) { + foreach($htmlContent->find('*') as $element) { if(in_array($element->tag, $text_to_keep)) { $element->outertext = $element->plaintext; } elseif(in_array($element->tag, $tags_to_remove)) { @@ -89,18 +79,7 @@ function backgroundToImg($htmlContent) { $regex = '/background-image[ ]{0,}:[ ]{0,}url\([\'"]{0,}(.*?)[\'"]{0,}\)/'; $htmlContent = str_get_html($htmlContent); - /* - * Notice: simple_html_dom currently doesn't support "->find(*)", which is a - * known issue: https://sourceforge.net/p/simplehtmldom/bugs/157/ - * - * A solution to this is to find all nodes WITHOUT a specific attribute. If - * the attribute is very unlikely to appear in the DOM, this is essentially - * returning all nodes. - * - * "*[!b38fd2b1fe7f4747d6b1c1254ccd055e]" is doing exactly that. The attrib - * "b38fd2b1fe7f4747d6b1c1254ccd055e" is very unlikely to appear in any DOM. - */ - foreach($htmlContent->find('*[!b38fd2b1fe7f4747d6b1c1254ccd055e]') as $element) { + foreach($htmlContent->find('*') as $element) { if(preg_match($regex, $element->style, $matches) > 0) { diff --git a/lib/rssbridge.php b/lib/rssbridge.php index dbeab26..a025f22 100644 --- a/lib/rssbridge.php +++ b/lib/rssbridge.php @@ -15,49 +15,65 @@ define('PATH_ROOT', __DIR__ . '/../'); /** Path to the core library */ -define('PATH_LIB', __DIR__ . '/../lib/'); // Path to core library +define('PATH_LIB', PATH_ROOT . 'lib/'); /** Path to the vendor library */ -define('PATH_LIB_VENDOR', __DIR__ . '/../vendor/'); +define('PATH_LIB_VENDOR', PATH_ROOT . 'vendor/'); /** Path to the bridges library */ -define('PATH_LIB_BRIDGES', __DIR__ . '/../bridges/'); +define('PATH_LIB_BRIDGES', PATH_ROOT . 'bridges/'); /** Path to the formats library */ -define('PATH_LIB_FORMATS', __DIR__ . '/../formats/'); +define('PATH_LIB_FORMATS', PATH_ROOT . 'formats/'); /** Path to the caches library */ -define('PATH_LIB_CACHES', __DIR__ . '/../caches/'); +define('PATH_LIB_CACHES', PATH_ROOT . 'caches/'); + +/** Path to the actions library */ +define('PATH_LIB_ACTIONS', PATH_ROOT . 'actions/'); /** Path to the cache folder */ -define('PATH_CACHE', __DIR__ . '/../cache/'); +define('PATH_CACHE', PATH_ROOT . 'cache/'); /** Path to the whitelist file */ -define('WHITELIST', __DIR__ . '/../whitelist.txt'); +define('WHITELIST', PATH_ROOT . 'whitelist.txt'); + +/** Path to the default whitelist file */ +define('WHITELIST_DEFAULT', PATH_ROOT . 'whitelist.default.txt'); + +/** Path to the configuration file */ +define('FILE_CONFIG', PATH_ROOT . 'config.ini.php'); + +/** Path to the default configuration file */ +define('FILE_CONFIG_DEFAULT', PATH_ROOT . 'config.default.ini.php'); /** URL to the RSS-Bridge repository */ define('REPOSITORY', 'https://github.com/RSS-Bridge/rss-bridge/'); // Interfaces +require_once PATH_LIB . 'ActionInterface.php'; require_once PATH_LIB . 'BridgeInterface.php'; require_once PATH_LIB . 'CacheInterface.php'; require_once PATH_LIB . 'FormatInterface.php'; // Classes +require_once PATH_LIB . 'FactoryAbstract.php'; require_once PATH_LIB . 'FeedItem.php'; require_once PATH_LIB . 'Debug.php'; require_once PATH_LIB . 'Exceptions.php'; -require_once PATH_LIB . 'Format.php'; +require_once PATH_LIB . 'FormatFactory.php'; require_once PATH_LIB . 'FormatAbstract.php'; -require_once PATH_LIB . 'Bridge.php'; +require_once PATH_LIB . 'BridgeFactory.php'; require_once PATH_LIB . 'BridgeAbstract.php'; require_once PATH_LIB . 'FeedExpander.php'; -require_once PATH_LIB . 'Cache.php'; +require_once PATH_LIB . 'CacheFactory.php'; require_once PATH_LIB . 'Authentication.php'; require_once PATH_LIB . 'Configuration.php'; require_once PATH_LIB . 'BridgeCard.php'; require_once PATH_LIB . 'BridgeList.php'; require_once PATH_LIB . 'ParameterValidator.php'; +require_once PATH_LIB . 'ActionFactory.php'; +require_once PATH_LIB . 'ActionAbstract.php'; // Functions require_once PATH_LIB . 'html.php'; @@ -65,16 +81,6 @@ require_once PATH_LIB . 'error.php'; require_once PATH_LIB . 'contents.php'; // Vendor +define('MAX_FILE_SIZE', 10000000); /* Allow larger files for simple_html_dom */ require_once PATH_LIB_VENDOR . 'simplehtmldom/simple_html_dom.php'; require_once PATH_LIB_VENDOR . 'php-urljoin/src/urljoin.php'; - -// Initialize static members -try { - Bridge::setWorkingDir(PATH_LIB_BRIDGES); - Format::setWorkingDir(PATH_LIB_FORMATS); - Cache::setWorkingDir(PATH_LIB_CACHES); -} catch(Exception $e) { - error_log($e); - header('Content-type: text/plain', true, 500); - die($e->getMessage()); -} diff --git a/static/HtmlFormat.css b/static/HtmlFormat.css index e17e325..4ebc79e 100644 --- a/static/HtmlFormat.css +++ b/static/HtmlFormat.css @@ -32,7 +32,6 @@ html, body, div, span, applet, object, iframe, h1, h2, h3, h4, h5, h6, p, blockq width: 60%; margin: 30px auto; padding: 15px 15px; - text-align: center; box-shadow: 0 6px 15px rgba(0, 0, 0, 0.09); border-radius: 4px; } @@ -59,6 +58,18 @@ html, body, div, span, applet, object, iframe, h1, h2, h3, h4, h5, h6, p, blockq } section > div.content, section > div.attachments { padding: 10px; +} + section h1, section h2, section h3, section b, section strong { + font-weight: bold; +} + section i, section em { + font-style: italic; +} + section p:not(:last-child) { + margin-bottom: 1em; +} + section li { + margin-left: 1em; } section > div.attachments > li.enclosure { list-style-type: circle; @@ -84,4 +95,21 @@ html, body, div, span, applet, object, iframe, h1, h2, h3, h4, h5, h6, p, blockq } button:hover { background: #49afff; -} \ No newline at end of file +} + +@media screen and (max-width: 767px) { + + section { + width: 100%; + padding: 0; + + } + + button { + display: inline-block; + width: 40%; + padding: 5px auto; + margin: 3px auto 0; + } + +} diff --git a/static/favicon.png b/static/favicon.png new file mode 100644 index 0000000..0a072e0 Binary files /dev/null and b/static/favicon.png differ diff --git a/static/favicon.svg b/static/favicon.svg new file mode 100644 index 0000000..0870bf6 --- /dev/null +++ b/static/favicon.svg @@ -0,0 +1,122 @@ + + + + + + + + + + image/svg+xml + + + + + + + + + Bridge + rss + + + + diff --git a/static/logo.svg b/static/logo.svg new file mode 100644 index 0000000..0096d79 --- /dev/null +++ b/static/logo.svg @@ -0,0 +1,162 @@ + + + + + + + + + + image/svg+xml + + + + + + + + reconnecting the web + Bridge + + rss + + + + + + + + + + + diff --git a/static/logo_300px.png b/static/logo_300px.png new file mode 100644 index 0000000..87a4ba4 Binary files /dev/null and b/static/logo_300px.png differ diff --git a/static/logo_600px.png b/static/logo_600px.png new file mode 100644 index 0000000..47660dc Binary files /dev/null and b/static/logo_600px.png differ diff --git a/static/style.css b/static/style.css index fa70f2d..974aa78 100644 --- a/static/style.css +++ b/static/style.css @@ -43,14 +43,11 @@ header { color: #1182DB; } -header > h1 { - font-size: 500%; - font-weight: bold; -} - -header > h2 { - margin-left: 1em; - font-size: 200%; +header > div.logo { + background-image: url(logo_600px.png); + width: 599px; + height: 177px; + margin: auto; } header > section.warning { @@ -84,6 +81,12 @@ input[type="number"]:focus { border-color: #888; } +input:focus::-webkit-input-placeholder { opacity: 0; } +input:focus::-moz-placeholder { opacity: 0; } +input:focus::placeholder { opacity: 0; } +input:focus:-moz-placeholder { opacity: 0; } +input:focus:-ms-input-placeholder { opacity: 0; } + .searchbar { width: 40%; margin: 40px auto 100px; @@ -101,13 +104,6 @@ input[type="number"]:focus { text-align: center; } -.searchbar input[type="text"]:focus::-webkit-input-placeholder, -.searchbar input[type="text"]:focus::-moz-placeholder, -.searchbar input[type="text"]:focus:-moz-placeholder, -.searchbar input[type="text"]:focus:-ms-input-placeholder { - opacity: 0; -} - .searchbar > h3 { font-size: 200%; font-weight: bold; @@ -200,6 +196,7 @@ form { .parameters label { text-align: right; + line-height: 1.5em; } .parameters label::before { @@ -304,3 +301,60 @@ h5 { .advice > li { text-align: left; } + +@media screen and (max-width: 767px) { + body { + font-size: 75%; + } + + header > div.logo { + background-image: url(logo_300px.png); + width: 300px; + height: 89px; + } + + header > section.warning { + width: 90%; + } + + header > section.critical-warning { + width: 90%; + } + + .searchbar { + width: 90%; + margin: 0 auto; + } + + section { + width: 90%; + margin: 10px auto; + overflow: hidden; + } + + button { + display: inline-block; + width: 40%; + padding: 5px auto; + margin: 3px auto 0; + } + + @supports (display: grid) { + + .parameters { + grid-template-columns: auto auto; + grid-column-gap: 5px; + } + + .parameters label { + line-height: 2em; + word-break: break-word; + } + + } /* @supports (display: grid) */ + + .secure-warning { + width: 100%; + } + +} \ No newline at end of file diff --git a/vendor/simplehtmldom/LICENSE b/vendor/simplehtmldom/LICENSE new file mode 100644 index 0000000..6040f77 --- /dev/null +++ b/vendor/simplehtmldom/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2019 S.C. Chen, John Schlick, logmanoriginal + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/vendor/simplehtmldom/simple_html_dom.php b/vendor/simplehtmldom/simple_html_dom.php index 43a1172..d30b018 100644 --- a/vendor/simplehtmldom/simple_html_dom.php +++ b/vendor/simplehtmldom/simple_html_dom.php @@ -1,188 +1,140 @@ size is the "real" number of bytes the dom was created from. - * but for most purposes, it's a really good estimation. - * Paperg - Added the forceTagsClosed to the dom constructor. Forcing tags closed is great for malformed html, but it CAN lead to parsing errors. - * Allow the user to tell us how much they trust the html. - * Paperg add the text and plaintext to the selectors for the find syntax. plaintext implies text in the innertext of a node. text implies that the tag is a text node. - * This allows for us to find tags based on the text they contain. - * Create find_ancestor_tag to see if a tag is - at any level - inside of another specific tag. - * Paperg: added parse_charset so that we know about the character set of the source document. - * NOTE: If the user's system has a routine called get_last_retrieve_url_contents_content_type availalbe, we will assume it's returning the content-type header from the - * last transfer or curl_exec, and we will parse that and use it in preference to any other method of charset detection. + * Licensed under The MIT License + * See the LICENSE file in the project root for more information. * - * Found infinite loop in the case of broken html in restore_noise. Rewrote to protect from that. - * PaperG (John Schlick) Added get_display_size for "IMG" tags. + * Authors: + * S.C. Chen + * John Schlick + * Rus Carroll + * logmanoriginal * - * Licensed under The MIT License - * Redistributions of files must retain the above copyright notice. + * Contributors: + * Yousuke Kumakura + * Vadim Voituk + * Antcs * - * @author S.C. Chen - * @author John Schlick - * @author Rus Carroll - * @version Rev. 1.7 (214) - * @package PlaceLocalInclude - * @subpackage simple_html_dom + * Version Rev. 1.9 (290) */ -/** - * All of the Defines for the classes below. - * @author S.C. Chen - */ define('HDOM_TYPE_ELEMENT', 1); define('HDOM_TYPE_COMMENT', 2); -define('HDOM_TYPE_TEXT', 3); -define('HDOM_TYPE_ENDTAG', 4); -define('HDOM_TYPE_ROOT', 5); +define('HDOM_TYPE_TEXT', 3); +define('HDOM_TYPE_ENDTAG', 4); +define('HDOM_TYPE_ROOT', 5); define('HDOM_TYPE_UNKNOWN', 6); define('HDOM_QUOTE_DOUBLE', 0); define('HDOM_QUOTE_SINGLE', 1); -define('HDOM_QUOTE_NO', 3); -define('HDOM_INFO_BEGIN', 0); -define('HDOM_INFO_END', 1); -define('HDOM_INFO_QUOTE', 2); -define('HDOM_INFO_SPACE', 3); -define('HDOM_INFO_TEXT', 4); -define('HDOM_INFO_INNER', 5); -define('HDOM_INFO_OUTER', 6); -define('HDOM_INFO_ENDSPACE',7); -define('DEFAULT_TARGET_CHARSET', 'UTF-8'); -define('DEFAULT_BR_TEXT', "\r\n"); -define('DEFAULT_SPAN_TEXT', " "); -define('MAX_FILE_SIZE', 10000000); - -/** Contents between curly braces "{" and "}" are interpreted as text */ +define('HDOM_QUOTE_NO', 3); +define('HDOM_INFO_BEGIN', 0); +define('HDOM_INFO_END', 1); +define('HDOM_INFO_QUOTE', 2); +define('HDOM_INFO_SPACE', 3); +define('HDOM_INFO_TEXT', 4); +define('HDOM_INFO_INNER', 5); +define('HDOM_INFO_OUTER', 6); +define('HDOM_INFO_ENDSPACE', 7); + +defined('DEFAULT_TARGET_CHARSET') || define('DEFAULT_TARGET_CHARSET', 'UTF-8'); +defined('DEFAULT_BR_TEXT') || define('DEFAULT_BR_TEXT', "\r\n"); +defined('DEFAULT_SPAN_TEXT') || define('DEFAULT_SPAN_TEXT', ' '); +defined('MAX_FILE_SIZE') || define('MAX_FILE_SIZE', 600000); define('HDOM_SMARTY_AS_TEXT', 1); -// helper functions -// ----------------------------------------------------------------------------- -// get html dom from file -// $maxlen is defined in the code as PHP_STREAM_COPY_ALL which is defined as -1. -function file_get_html($url, $use_include_path = false, $context=null, $offset = 0, $maxLen=-1, $lowercase = true, $forceTagsClosed=true, $target_charset = DEFAULT_TARGET_CHARSET, $stripRN=true, $defaultBRText=DEFAULT_BR_TEXT, $defaultSpanText=DEFAULT_SPAN_TEXT) +function file_get_html( + $url, + $use_include_path = false, + $context = null, + $offset = 0, + $maxLen = -1, + $lowercase = true, + $forceTagsClosed = true, + $target_charset = DEFAULT_TARGET_CHARSET, + $stripRN = true, + $defaultBRText = DEFAULT_BR_TEXT, + $defaultSpanText = DEFAULT_SPAN_TEXT) { - // Ensure maximum length is greater than zero if($maxLen <= 0) { $maxLen = MAX_FILE_SIZE; } - // We DO force the tags to be terminated. - $dom = new simple_html_dom(null, $lowercase, $forceTagsClosed, $target_charset, $stripRN, $defaultBRText, $defaultSpanText); - // For sourceforge users: uncomment the next line and comment the retrieve_url_contents line 2 lines down if it is not already done. - $contents = file_get_contents($url, $use_include_path, $context, $offset, $maxLen); - // Paperg - use our own mechanism for getting the contents as we want to control the timeout. - //$contents = retrieve_url_contents($url); - if (empty($contents) || strlen($contents) > $maxLen) - { + $dom = new simple_html_dom( + null, + $lowercase, + $forceTagsClosed, + $target_charset, + $stripRN, + $defaultBRText, + $defaultSpanText + ); + + /** + * For sourceforge users: uncomment the next line and comment the + * retrieve_url_contents line 2 lines down if it is not already done. + */ + $contents = file_get_contents( + $url, + $use_include_path, + $context, + $offset, + $maxLen + ); + // $contents = retrieve_url_contents($url); + + if (empty($contents) || strlen($contents) > $maxLen) { + $dom->clear(); return false; } - // The second parameter can force the selectors to all be lowercase. - $dom->load($contents, $lowercase, $stripRN); - return $dom; + + return $dom->load($contents, $lowercase, $stripRN); } -// get html dom from string -function str_get_html($str, $lowercase=true, $forceTagsClosed=true, $target_charset = DEFAULT_TARGET_CHARSET, $stripRN=true, $defaultBRText=DEFAULT_BR_TEXT, $defaultSpanText=DEFAULT_SPAN_TEXT) +function str_get_html( + $str, + $lowercase = true, + $forceTagsClosed = true, + $target_charset = DEFAULT_TARGET_CHARSET, + $stripRN = true, + $defaultBRText = DEFAULT_BR_TEXT, + $defaultSpanText = DEFAULT_SPAN_TEXT) { - $dom = new simple_html_dom(null, $lowercase, $forceTagsClosed, $target_charset, $stripRN, $defaultBRText, $defaultSpanText); - if (empty($str) || strlen($str) > MAX_FILE_SIZE) - { + $dom = new simple_html_dom( + null, + $lowercase, + $forceTagsClosed, + $target_charset, + $stripRN, + $defaultBRText, + $defaultSpanText + ); + + if (empty($str) || strlen($str) > MAX_FILE_SIZE) { $dom->clear(); return false; } - $dom->load($str, $lowercase, $stripRN); - return $dom; + + return $dom->load($str, $lowercase, $stripRN); } -// dump html dom tree -function dump_html_tree($node, $show_attr=true, $deep=0) +function dump_html_tree($node, $show_attr = true, $deep = 0) { $node->dump($node); } - -/** - * simple html dom node - * PaperG - added ability for "find" routine to lowercase the value of the selector. - * PaperG - added $tag_start to track the start position of the tag in the total byte index - * - * @package PlaceLocalInclude - */ class simple_html_dom_node { - /** - * Node type - * - * Default is {@see HDOM_TYPE_TEXT} - * - * @var int - */ public $nodetype = HDOM_TYPE_TEXT; - - /** - * Tag name - * - * Default is 'text' - * - * @var string - */ public $tag = 'text'; - - /** - * List of attributes - * - * @var array - */ public $attr = array(); - - /** - * List of child node objects - * - * @var array - */ public $children = array(); public $nodes = array(); - - /** - * The parent node object - * - * @var object|null - */ public $parent = null; - - // The "info" array - see HDOM_INFO_... for what each element contains. public $_ = array(); - - /** - * Start position of the tag in the document - * - * @var int - */ public $tag_start = 0; - - /** - * The DOM object - * - * @var object|null - */ private $dom = null; - /** - * Construct new node object - * - * Adds itself to the list of DOM Nodes {@see simple_html_dom::$nodes} - */ function __construct($dom) { $this->dom = $dom; @@ -199,7 +151,6 @@ class simple_html_dom_node return $this->outertext(); } - // clean up memory due to php5 circular references memory leak... function clear() { $this->dom = null; @@ -208,109 +159,86 @@ class simple_html_dom_node $this->children = null; } - // dump node's tree - function dump($show_attr=true, $deep=0) + function dump($show_attr = true, $depth = 0) { - $lead = str_repeat(' ', $deep); + echo str_repeat("\t", $depth) . $this->tag; - echo $lead.$this->tag; - if ($show_attr && count($this->attr)>0) - { + if ($show_attr && count($this->attr) > 0) { echo '('; - foreach ($this->attr as $k=>$v) - echo "[$k]=>\"".$this->$k.'", '; + foreach ($this->attr as $k => $v) { + echo "[$k]=>\"$v\", "; + } echo ')'; } + echo "\n"; - if ($this->nodes) - { - foreach ($this->nodes as $c) - { - $c->dump($show_attr, $deep+1); + if ($this->nodes) { + foreach ($this->nodes as $node) { + $node->dump($show_attr, $depth + 1); } } } - - // Debugging function to dump a single dom node with a bunch of information about it. - function dump_node($echo=true) + function dump_node($echo = true) { - $string = $this->tag; - if (count($this->attr)>0) - { + + if (count($this->attr) > 0) { $string .= '('; - foreach ($this->attr as $k=>$v) - { - $string .= "[$k]=>\"".$this->$k.'", '; + foreach ($this->attr as $k => $v) { + $string .= "[$k]=>\"$v\", "; } $string .= ')'; } - if (count($this->_)>0) - { + + if (count($this->_) > 0) { $string .= ' $_ ('; - foreach ($this->_ as $k=>$v) - { - if (is_array($v)) - { + foreach ($this->_ as $k => $v) { + if (is_array($v)) { $string .= "[$k]=>("; - foreach ($v as $k2=>$v2) - { - $string .= "[$k2]=>\"".$v2.'", '; + foreach ($v as $k2 => $v2) { + $string .= "[$k2]=>\"$v2\", "; } - $string .= ")"; + $string .= ')'; } else { - $string .= "[$k]=>\"".$v.'", '; + $string .= "[$k]=>\"$v\", "; } } - $string .= ")"; + $string .= ')'; } - if (isset($this->text)) - { - $string .= " text: (" . $this->text . ")"; + if (isset($this->text)) { + $string .= " text: ({$this->text})"; } - $string .= " HDOM_INNER_INFO: '"; - if (isset($node->_[HDOM_INFO_INNER])) - { - $string .= $node->_[HDOM_INFO_INNER] . "'"; - } - else - { + $string .= ' HDOM_INNER_INFO: '; + + if (isset($node->_[HDOM_INFO_INNER])) { + $string .= "'" . $node->_[HDOM_INFO_INNER] . "'"; + } else { $string .= ' NULL '; } - $string .= " children: " . count($this->children); - $string .= " nodes: " . count($this->nodes); - $string .= " tag_start: " . $this->tag_start; + $string .= ' children: ' . count($this->children); + $string .= ' nodes: ' . count($this->nodes); + $string .= ' tag_start: ' . $this->tag_start; $string .= "\n"; - if ($echo) - { + if ($echo) { echo $string; return; - } - else - { + } else { return $string; } } - /** - * Return or set parent node - * - * @param object|null $parent (optional) The parent node, `null` to return - * the current parent node. - * @return object|null The parent node - */ - function parent($parent=null) + function parent($parent = null) { // I am SURE that this doesn't work properly. - // It fails to unset the current node from it's current parents nodes or children list first. - if ($parent !== null) - { + // It fails to unset the current node from it's current parents nodes or + // children list first. + if ($parent !== null) { $this->parent = $parent; $this->parent->nodes[] = $this; $this->parent->children[] = $this; @@ -319,272 +247,214 @@ class simple_html_dom_node return $this->parent; } - /** - * @return bool True if the node has at least one child node - */ function has_child() { return !empty($this->children); } - /** - * Get child node at specified index - * - * @param int $idx The index of the child node to return, `-1` to return all - * child nodes. - * @return object|array|null The child node at the specified index, all child - * nodes or null if the index is invalid. - */ - function children($idx=-1) + function children($idx = -1) { - if ($idx===-1) - { + if ($idx === -1) { return $this->children; } - if (isset($this->children[$idx])) - { + + if (isset($this->children[$idx])) { return $this->children[$idx]; } + return null; } - /** - * Get first child node - * - * @return object|null The first child node or null if the current node has - * no child nodes. - * - * @todo Use `empty()` instead of `count()` to improve performance on large - * arrays. - */ function first_child() { - if (count($this->children)>0) - { + if (count($this->children) > 0) { return $this->children[0]; } return null; } - /** - * Get last child node - * - * @return object|null The last child node or null if the current node has - * no child nodes. - * - * @todo Use `end()` to slightly improve performance on large arrays. - */ function last_child() { - if (($count=count($this->children))>0) - { - return $this->children[$count-1]; + if (count($this->children) > 0) { + return end($this->children); } return null; } - /** - * Get next sibling node - * - * @return object|null The sibling node or null if the current node has no - * sibling nodes. - */ function next_sibling() { - if ($this->parent===null) - { + if ($this->parent === null) { return null; } - $idx = 0; - $count = count($this->parent->children); - while ($idx<$count && $this!==$this->parent->children[$idx]) - { - ++$idx; - } - if (++$idx>=$count) - { - return null; + $idx = array_search($this, $this->parent->children, true); + + if ($idx !== false && isset($this->parent->children[$idx + 1])) { + return $this->parent->children[$idx + 1]; } - return $this->parent->children[$idx]; + + return null; } - /** - * Get previous sibling node - * - * @return object|null The sibling node or null if the current node has no - * sibling nodes. - */ function prev_sibling() { - if ($this->parent===null) return null; - $idx = 0; - $count = count($this->parent->children); - while ($idx<$count && $this!==$this->parent->children[$idx]) - ++$idx; - if (--$idx<0) return null; - return $this->parent->children[$idx]; + if ($this->parent === null) { + return null; + } + + $idx = array_search($this, $this->parent->children, true); + + if ($idx !== false && $idx > 0) { + return $this->parent->children[$idx - 1]; + } + + return null; } - /** - * Traverse ancestors to the first matching tag. - * - * @param string $tag Tag to find - * @return object|null First matching node in the DOM tree or null if no - * match was found. - * - * @todo Null is returned implicitly by calling ->parent on the root node. - * This behaviour could change at any time, rendering this function invalid. - */ function find_ancestor_tag($tag) { global $debug_object; if (is_object($debug_object)) { $debug_object->debug_log_entry(1); } - // Start by including ourselves in the comparison. - $returnDom = $this; + if ($this->parent === null) { + return null; + } + + $ancestor = $this->parent; - while (!is_null($returnDom)) - { - if (is_object($debug_object)) { $debug_object->debug_log(2, "Current tag is: " . $returnDom->tag); } + while (!is_null($ancestor)) { + if (is_object($debug_object)) { + $debug_object->debug_log(2, 'Current tag is: ' . $ancestor->tag); + } - if ($returnDom->tag == $tag) - { + if ($ancestor->tag === $tag) { break; } - $returnDom = $returnDom->parent; + + $ancestor = $ancestor->parent; } - return $returnDom; + + return $ancestor; } - /** - * Get node's inner text (everything inside the opening and closing tags) - * - * @return string - */ function innertext() { - if (isset($this->_[HDOM_INFO_INNER])) return $this->_[HDOM_INFO_INNER]; - if (isset($this->_[HDOM_INFO_TEXT])) return $this->dom->restore_noise($this->_[HDOM_INFO_TEXT]); + if (isset($this->_[HDOM_INFO_INNER])) { + return $this->_[HDOM_INFO_INNER]; + } + + if (isset($this->_[HDOM_INFO_TEXT])) { + return $this->dom->restore_noise($this->_[HDOM_INFO_TEXT]); + } $ret = ''; - foreach ($this->nodes as $n) + + foreach ($this->nodes as $n) { $ret .= $n->outertext(); + } + return $ret; } - /** - * Get node's outer text (everything including the opening and closing tags) - * - * @return string - */ function outertext() { global $debug_object; - if (is_object($debug_object)) - { + + if (is_object($debug_object)) { $text = ''; - if ($this->tag == 'text') - { - if (!empty($this->text)) - { - $text = " with text: " . $this->text; + + if ($this->tag === 'text') { + if (!empty($this->text)) { + $text = ' with text: ' . $this->text; } } + $debug_object->debug_log(1, 'Innertext of tag: ' . $this->tag . $text); } - if ($this->tag==='root') return $this->innertext(); + if ($this->tag === 'root') { + return $this->innertext(); + } - // trigger callback - if ($this->dom && $this->dom->callback!==null) - { + // todo: What is the use of this callback? Remove? + if ($this->dom && $this->dom->callback !== null) { call_user_func_array($this->dom->callback, array($this)); } - if (isset($this->_[HDOM_INFO_OUTER])) return $this->_[HDOM_INFO_OUTER]; - if (isset($this->_[HDOM_INFO_TEXT])) return $this->dom->restore_noise($this->_[HDOM_INFO_TEXT]); + if (isset($this->_[HDOM_INFO_OUTER])) { + return $this->_[HDOM_INFO_OUTER]; + } + + if (isset($this->_[HDOM_INFO_TEXT])) { + return $this->dom->restore_noise($this->_[HDOM_INFO_TEXT]); + } + + $ret = ''; - // render begin tag - if ($this->dom && $this->dom->nodes[$this->_[HDOM_INFO_BEGIN]]) - { + if ($this->dom && $this->dom->nodes[$this->_[HDOM_INFO_BEGIN]]) { $ret = $this->dom->nodes[$this->_[HDOM_INFO_BEGIN]]->makeup(); - } else { - $ret = ""; } - // render inner text - if (isset($this->_[HDOM_INFO_INNER])) - { - // If it's a br tag... don't return the HDOM_INNER_INFO that we may or may not have added. - if ($this->tag != "br") - { + if (isset($this->_[HDOM_INFO_INNER])) { + // todo:
should either never have HDOM_INFO_INNER or always + if ($this->tag !== 'br') { $ret .= $this->_[HDOM_INFO_INNER]; } - } else { - if ($this->nodes) - { - foreach ($this->nodes as $n) - { - $ret .= $this->convert_text($n->outertext()); - } + } elseif ($this->nodes) { + foreach ($this->nodes as $n) { + $ret .= $this->convert_text($n->outertext()); } } - // render end tag - if (isset($this->_[HDOM_INFO_END]) && $this->_[HDOM_INFO_END]!=0) - $ret .= 'tag.'>'; + if (isset($this->_[HDOM_INFO_END]) && $this->_[HDOM_INFO_END] != 0) { + $ret .= 'tag . '>'; + } + return $ret; } - /** - * Get node's plain text (everything excluding all tags) - * - * @return string - */ function text() { - if (isset($this->_[HDOM_INFO_INNER])) return $this->_[HDOM_INFO_INNER]; - switch ($this->nodetype) - { + if (isset($this->_[HDOM_INFO_INNER])) { + return $this->_[HDOM_INFO_INNER]; + } + + switch ($this->nodetype) { case HDOM_TYPE_TEXT: return $this->dom->restore_noise($this->_[HDOM_INFO_TEXT]); case HDOM_TYPE_COMMENT: return ''; case HDOM_TYPE_UNKNOWN: return ''; } - if (strcasecmp($this->tag, 'script')===0) return ''; - if (strcasecmp($this->tag, 'style')===0) return ''; + + if (strcasecmp($this->tag, 'script') === 0) { return ''; } + if (strcasecmp($this->tag, 'style') === 0) { return ''; } $ret = ''; - // In rare cases, (always node type 1 or HDOM_TYPE_ELEMENT - observed for some span tags, and some p tags) $this->nodes is set to NULL. - // NOTE: This indicates that there is a problem where it's set to NULL without a clear happening. + + // In rare cases, (always node type 1 or HDOM_TYPE_ELEMENT - observed + // for some span tags, and some p tags) $this->nodes is set to NULL. + // NOTE: This indicates that there is a problem where it's set to NULL + // without a clear happening. // WHY is this happening? - if (!is_null($this->nodes)) - { - foreach ($this->nodes as $n) - { + if (!is_null($this->nodes)) { + foreach ($this->nodes as $n) { // Start paragraph after a blank line - if ($n->tag == 'p') - { - $ret .= "\n\n"; + if ($n->tag === 'p') { + $ret = trim($ret) . "\n\n"; } $ret .= $this->convert_text($n->text()); - // If this node is a span... add a space at the end of it so multiple spans don't run into each other. This is plaintext after all. - if ($n->tag == "span") - { + // If this node is a span... add a space at the end of it so + // multiple spans don't run into each other. This is plaintext + // after all. + if ($n->tag === 'span') { $ret .= $this->dom->default_span_text; } } } - return trim($ret); + return $ret; } - /** - * Get node's xml text (inner text as a CDATA section) - * - * @return string - */ function xmltext() { $ret = $this->innertext(); @@ -593,76 +463,82 @@ class simple_html_dom_node return $ret; } - // build node's text with tag function makeup() { // text, comment, unknown - if (isset($this->_[HDOM_INFO_TEXT])) return $this->dom->restore_noise($this->_[HDOM_INFO_TEXT]); + if (isset($this->_[HDOM_INFO_TEXT])) { + return $this->dom->restore_noise($this->_[HDOM_INFO_TEXT]); + } - $ret = '<'.$this->tag; + $ret = '<' . $this->tag; $i = -1; - foreach ($this->attr as $key=>$val) - { + foreach ($this->attr as $key => $val) { ++$i; // skip removed attribute - if ($val===null || $val===false) - continue; + if ($val === null || $val === false) { continue; } $ret .= $this->_[HDOM_INFO_SPACE][$i][0]; + //no value attr: nowrap, checked selected... - if ($val===true) + if ($val === true) { $ret .= $key; - else { + } else { switch ($this->_[HDOM_INFO_QUOTE][$i]) { case HDOM_QUOTE_DOUBLE: $quote = '"'; break; case HDOM_QUOTE_SINGLE: $quote = '\''; break; default: $quote = ''; } - $ret .= $key.$this->_[HDOM_INFO_SPACE][$i][1].'='.$this->_[HDOM_INFO_SPACE][$i][2].$quote.$val.$quote; + + $ret .= $key + . $this->_[HDOM_INFO_SPACE][$i][1] + . '=' + . $this->_[HDOM_INFO_SPACE][$i][2] + . $quote + . $val + . $quote; } } + $ret = $this->dom->restore_noise($ret); return $ret . $this->_[HDOM_INFO_ENDSPACE] . '>'; } - // find elements by css selector - //PaperG - added ability for find to lowercase the value of the selector. - function find($selector, $idx=null, $lowercase=false) + function find($selector, $idx = null, $lowercase = false) { $selectors = $this->parse_selector($selector); - if (($count=count($selectors))===0) return array(); + if (($count = count($selectors)) === 0) { return array(); } $found_keys = array(); // find each selector - for ($c=0; $c<$count; ++$c) - { - // The change on the below line was documented on the sourceforge code tracker id 2788009 + for ($c = 0; $c < $count; ++$c) { + // The change on the below line was documented on the sourceforge + // code tracker id 2788009 // used to be: if (($levle=count($selectors[0]))===0) return array(); - if (($levle=count($selectors[$c]))===0) return array(); - if (!isset($this->_[HDOM_INFO_BEGIN])) return array(); + if (($levle = count($selectors[$c])) === 0) { return array(); } + if (!isset($this->_[HDOM_INFO_BEGIN])) { return array(); } - $head = array($this->_[HDOM_INFO_BEGIN]=>1); + $head = array($this->_[HDOM_INFO_BEGIN] => 1); + $cmd = ' '; // Combinator // handle descendant selectors, no recursive! - for ($l=0; $l<$levle; ++$l) - { + for ($l = 0; $l < $levle; ++$l) { $ret = array(); - foreach ($head as $k=>$v) - { - $n = ($k===-1) ? $this->dom->root : $this->dom->nodes[$k]; + + foreach ($head as $k => $v) { + $n = ($k === -1) ? $this->dom->root : $this->dom->nodes[$k]; //PaperG - Pass this optional parameter on to the seek function. - $n->seek($selectors[$c][$l], $ret, $lowercase); + $n->seek($selectors[$c][$l], $ret, $cmd, $lowercase); } + $head = $ret; + $cmd = $selectors[$c][$l][4]; // Next Combinator } - foreach ($head as $k=>$v) - { - if (!isset($found_keys[$k])) - { + foreach ($head as $k => $v) { + if (!isset($found_keys[$k])) { $found_keys[$k] = 1; } } @@ -672,192 +548,421 @@ class simple_html_dom_node ksort($found_keys); $found = array(); - foreach ($found_keys as $k=>$v) + foreach ($found_keys as $k => $v) { $found[] = $this->dom->nodes[$k]; + } // return nth-element or array - if (is_null($idx)) return $found; - else if ($idx<0) $idx = count($found) + $idx; + if (is_null($idx)) { return $found; } + elseif ($idx < 0) { $idx = count($found) + $idx; } return (isset($found[$idx])) ? $found[$idx] : null; } - // seek for given conditions - // PaperG - added parameter to allow for case insensitive testing of the value of a selector. - protected function seek($selector, &$ret, $lowercase=false) + protected function seek($selector, &$ret, $parent_cmd, $lowercase = false) { global $debug_object; if (is_object($debug_object)) { $debug_object->debug_log_entry(1); } - list($tag, $key, $val, $exp, $no_key) = $selector; - - // xpath index - if ($tag && $key && is_numeric($key)) - { - $count = 0; - foreach ($this->children as $c) - { - if ($tag==='*' || $tag===$c->tag) { - if (++$count==$key) { - $ret[$c->_[HDOM_INFO_BEGIN]] = 1; - return; - } + list($tag, $id, $class, $attributes, $cmb) = $selector; + $nodes = array(); + + if ($parent_cmd === ' ') { // Descendant Combinator + // Find parent closing tag if the current element doesn't have a closing + // tag (i.e. void element) + $end = (!empty($this->_[HDOM_INFO_END])) ? $this->_[HDOM_INFO_END] : 0; + if ($end == 0) { + $parent = $this->parent; + while (!isset($parent->_[HDOM_INFO_END]) && $parent !== null) { + $end -= 1; + $parent = $parent->parent; } + $end += $parent->_[HDOM_INFO_END]; } - return; - } - $end = (!empty($this->_[HDOM_INFO_END])) ? $this->_[HDOM_INFO_END] : 0; - if ($end==0) { - $parent = $this->parent; - while (!isset($parent->_[HDOM_INFO_END]) && $parent!==null) { - $end -= 1; - $parent = $parent->parent; + // Get list of target nodes + $nodes_start = $this->_[HDOM_INFO_BEGIN] + 1; + $nodes_count = $end - $nodes_start; + $nodes = array_slice($this->dom->nodes, $nodes_start, $nodes_count, true); + } elseif ($parent_cmd === '>') { // Child Combinator + $nodes = $this->children; + } elseif ($parent_cmd === '+' + && $this->parent + && in_array($this, $this->parent->children)) { // Next-Sibling Combinator + $index = array_search($this, $this->parent->children, true) + 1; + if ($index < count($this->parent->children)) + $nodes[] = $this->parent->children[$index]; + } elseif ($parent_cmd === '~' + && $this->parent + && in_array($this, $this->parent->children)) { // Subsequent Sibling Combinator + $index = array_search($this, $this->parent->children, true); + $nodes = array_slice($this->parent->children, $index); + } + + // Go throgh each element starting at this element until the end tag + // Note: If this element is a void tag, any previous void element is + // skipped. + foreach($nodes as $node) { + $pass = true; + + // Skip root nodes + if(!$node->parent) { + $pass = false; } - $end += $parent->_[HDOM_INFO_END]; - } - for ($i=$this->_[HDOM_INFO_BEGIN]+1; $i<$end; ++$i) { - $node = $this->dom->nodes[$i]; + // Skip if node isn't a child node (i.e. text nodes) + if($pass && !in_array($node, $node->parent->children, true)) { + $pass = false; + } - $pass = true; + // Skip if tag doesn't match + if ($pass && $tag !== '' && $tag !== $node->tag && $tag !== '*') { + $pass = false; + } - if ($tag==='*' && !$key) { - if (in_array($node, $this->children, true)) - $ret[$i] = 1; - continue; + // Skip if ID doesn't exist + if ($pass && $id !== '' && !isset($node->attr['id'])) { + $pass = false; } - // compare tag - if ($tag && $tag!=$node->tag && $tag!=='*') {$pass=false;} - // compare key - if ($pass && $key) { - if ($no_key) { - if (isset($node->attr[$key])) $pass=false; - } else { - if (($key != "plaintext") && !isset($node->attr[$key])) $pass=false; - } + // Check if ID matches + if ($pass && $id !== '' && isset($node->attr['id'])) { + // Note: Only consider the first ID (as browsers do) + $node_id = explode(' ', trim($node->attr['id']))[0]; + + if($id !== $node_id) { $pass = false; } } - // compare value - if ($pass && $key && $val && $val!=='*') { - // If they have told us that this is a "plaintext" search then we want the plaintext of the node - right? - if ($key == "plaintext") { - // $node->plaintext actually returns $node->text(); - $nodeKeyValue = $node->text(); - } else { - // this is a normal search, we want the value of that attribute of the tag. - $nodeKeyValue = $node->attr[$key]; - } - if (is_object($debug_object)) {$debug_object->debug_log(2, "testing node: " . $node->tag . " for attribute: " . $key . $exp . $val . " where nodes value is: " . $nodeKeyValue);} - //PaperG - If lowercase is set, do a case insensitive test of the value of the selector. - if ($lowercase) { - $check = $this->match($exp, strtolower($val), strtolower($nodeKeyValue)); + // Check if all class(es) exist + if ($pass && $class !== '' && is_array($class) && !empty($class)) { + if (isset($node->attr['class'])) { + $node_classes = explode(' ', $node->attr['class']); + + if ($lowercase) { + $node_classes = array_map('strtolower', $node_classes); + } + + foreach($class as $c) { + if(!in_array($c, $node_classes)) { + $pass = false; + break; + } + } } else { - $check = $this->match($exp, $val, $nodeKeyValue); + $pass = false; } - if (is_object($debug_object)) {$debug_object->debug_log(2, "after match: " . ($check ? "true" : "false"));} - - // handle multiple class - if (!$check && strcasecmp($key, 'class')===0) { - foreach (explode(' ',$node->attr[$key]) as $k) { - // Without this, there were cases where leading, trailing, or double spaces lead to our comparing blanks - bad form. - if (!empty($k)) { - if ($lowercase) { - $check = $this->match($exp, strtolower($val), strtolower($k)); - } else { - $check = $this->match($exp, $val, $k); + } + + // Check attributes + if ($pass + && $attributes !== '' + && is_array($attributes) + && !empty($attributes)) { + foreach($attributes as $a) { + list ( + $att_name, + $att_expr, + $att_val, + $att_inv, + $att_case_sensitivity + ) = $a; + + // Handle indexing attributes (i.e. "[2]") + /** + * Note: This is not supported by the CSS Standard but adds + * the ability to select items compatible to XPath (i.e. + * the 3rd element within it's parent). + * + * Note: This doesn't conflict with the CSS Standard which + * doesn't work on numeric attributes anyway. + */ + if (is_numeric($att_name) + && $att_expr === '' + && $att_val === '') { + $count = 0; + + // Find index of current element in parent + foreach ($node->parent->children as $c) { + if ($c->tag === $node->tag) ++$count; + if ($c === $node) break; + } + + // If this is the correct node, continue with next + // attribute + if ($count === (int)$att_name) continue; + } + + // Check attribute availability + if ($att_inv) { // Attribute should NOT be set + if (isset($node->attr[$att_name])) { + $pass = false; + break; } - if ($check) break; + } else { // Attribute should be set + // todo: "plaintext" is not a valid CSS selector! + if ($att_name !== 'plaintext' + && !isset($node->attr[$att_name])) { + $pass = false; + break; + } + } + + // Continue with next attribute if expression isn't defined + if ($att_expr === '') continue; + + // If they have told us that this is a "plaintext" + // search then we want the plaintext of the node - right? + // todo "plaintext" is not a valid CSS selector! + if ($att_name === 'plaintext') { + $nodeKeyValue = $node->text(); + } else { + $nodeKeyValue = $node->attr[$att_name]; + } + + if (is_object($debug_object)) { + $debug_object->debug_log(2, + 'testing node: ' + . $node->tag + . ' for attribute: ' + . $att_name + . $att_expr + . $att_val + . ' where nodes value is: ' + . $nodeKeyValue + ); + } + + // If lowercase is set, do a case insensitive test of + // the value of the selector. + if ($lowercase) { + $check = $this->match( + $att_expr, + strtolower($att_val), + strtolower($nodeKeyValue), + $att_case_sensitivity + ); + } else { + $check = $this->match( + $att_expr, + $att_val, + $nodeKeyValue, + $att_case_sensitivity + ); + } + + if (is_object($debug_object)) { + $debug_object->debug_log(2, + 'after match: ' + . ($check ? 'true' : 'false') + ); + } + + if (!$check) { + $pass = false; + break; } } - } - if (!$check) $pass = false; } - if ($pass) $ret[$i] = 1; + + // Found a match. Add to list and clear node + if ($pass) $ret[$node->_[HDOM_INFO_BEGIN]] = 1; unset($node); } // It's passed by reference so this is actually what this function returns. - if (is_object($debug_object)) {$debug_object->debug_log(1, "EXIT - ret: ", $ret);} + if (is_object($debug_object)) { + $debug_object->debug_log(1, 'EXIT - ret: ', $ret); + } } - protected function match($exp, $pattern, $value) { + protected function match($exp, $pattern, $value, $case_sensitivity) + { global $debug_object; if (is_object($debug_object)) {$debug_object->debug_log_entry(1);} + if ($case_sensitivity === 'i') { + $pattern = strtolower($pattern); + $value = strtolower($value); + } + switch ($exp) { case '=': - return ($value===$pattern); + return ($value === $pattern); case '!=': - return ($value!==$pattern); + return ($value !== $pattern); case '^=': - return preg_match("/^".preg_quote($pattern,'/')."/", $value); + return preg_match('/^' . preg_quote($pattern, '/') . '/', $value); case '$=': - return preg_match("/".preg_quote($pattern,'/')."$/", $value); + return preg_match('/' . preg_quote($pattern, '/') . '$/', $value); case '*=': - if ($pattern[0]=='/') { - return preg_match($pattern, $value); - } - return preg_match("/".$pattern."/i", $value); + return preg_match('/' . preg_quote($pattern, '/') . '/', $value); + case '|=': + /** + * [att|=val] + * + * Represents an element with the att attribute, its value + * either being exactly "val" or beginning with "val" + * immediately followed by "-" (U+002D). + */ + return strpos($value, $pattern) === 0; + case '~=': + /** + * [att~=val] + * + * Represents an element with the att attribute whose value is a + * whitespace-separated list of words, one of which is exactly + * "val". If "val" contains whitespace, it will never represent + * anything (since the words are separated by spaces). Also if + * "val" is the empty string, it will never represent anything. + */ + return in_array($pattern, explode(' ', trim($value)), true); } return false; } - protected function parse_selector($selector_string) { + protected function parse_selector($selector_string) + { global $debug_object; - if (is_object($debug_object)) {$debug_object->debug_log_entry(1);} + if (is_object($debug_object)) { $debug_object->debug_log_entry(1); } - // pattern of CSS selectors, modified from mootools - // Paperg: Add the colon to the attrbute, so that it properly finds like google does. - // Note: if you try to look at this attribute, yo MUST use getAttribute since $dom->x:y will fail the php syntax check. -// Notice the \[ starting the attbute? and the @? following? This implies that an attribute can begin with an @ sign that is not captured. -// This implies that an html attribute specifier may start with an @ sign that is NOT captured by the expression. -// farther study is required to determine of this should be documented or removed. -// $pattern = "/([\w-:\*]*)(?:\#([\w-]+)|\.([\w-]+))?(?:\[@?(!?[\w-]+)(?:([!*^$]?=)[\"']?(.*?)[\"']?)?\])?([\/, ]+)/is"; - $pattern = "/([\w:\*-]*)(?:\#([\w-]+)|\.([\w-]+))?(?:\[@?(!?[\w:-]+)(?:([!*^$]?=)[\"']?(.*?)[\"']?)?\])?([\/, ]+)/is"; - preg_match_all($pattern, trim($selector_string).' ', $matches, PREG_SET_ORDER); - if (is_object($debug_object)) {$debug_object->debug_log(2, "Matches Array: ", $matches);} + /** + * Pattern of CSS selectors, modified from mootools (https://mootools.net/) + * + * Paperg: Add the colon to the attribute, so that it properly finds + * like google does. + * + * Note: if you try to look at this attribute, you MUST use getAttribute + * since $dom->x:y will fail the php syntax check. + * + * Notice the \[ starting the attribute? and the @? following? This + * implies that an attribute can begin with an @ sign that is not + * captured. This implies that an html attribute specifier may start + * with an @ sign that is NOT captured by the expression. Farther study + * is required to determine of this should be documented or removed. + * + * Matches selectors in this order: + * + * [0] - full match + * + * [1] - tag name + * ([\w:\*-]*) + * Matches the tag name consisting of zero or more words, colons, + * asterisks and hyphens. + * + * [2] - id name + * (?:\#([\w-]+)) + * Optionally matches a id name, consisting of an "#" followed by + * the id name (one or more words and hyphens). + * + * [3] - class names (including dots) + * (?:\.([\w\.-]+))? + * Optionally matches a list of classs, consisting of an "." + * followed by the class name (one or more words and hyphens) + * where multiple classes can be chained (i.e. ".foo.bar.baz") + * + * [4] - attributes + * ((?:\[@?(?:!?[\w:-]+)(?:(?:[!*^$|~]?=)[\"']?(?:.*?)[\"']?)?(?:\s*?(?:[iIsS])?)?\])+)? + * Optionally matches the attributes list + * + * [5] - separator + * ([\/, >+~]+) + * Matches the selector list separator + */ + // phpcs:ignore Generic.Files.LineLength + $pattern = "/([\w:\*-]*)(?:\#([\w-]+))?(?:|\.([\w\.-]+))?((?:\[@?(?:!?[\w:-]+)(?:(?:[!*^$|~]?=)[\"']?(?:.*?)[\"']?)?(?:\s*?(?:[iIsS])?)?\])+)?([\/, >+~]+)/is"; + + preg_match_all( + $pattern, + trim($selector_string) . ' ', // Add final ' ' as pseudo separator + $matches, + PREG_SET_ORDER + ); + + if (is_object($debug_object)) { + $debug_object->debug_log(2, 'Matches Array: ', $matches); + } $selectors = array(); $result = array(); - //print_r($matches); foreach ($matches as $m) { $m[0] = trim($m[0]); - if ($m[0]==='' || $m[0]==='/' || $m[0]==='//') continue; - // for browser generated xpath - if ($m[1]==='tbody') continue; - - list($tag, $key, $val, $exp, $no_key) = array($m[1], null, null, '=', false); - if (!empty($m[2])) {$key='id'; $val=$m[2];} - if (!empty($m[3])) {$key='class'; $val=$m[3];} - if (!empty($m[4])) {$key=$m[4];} - if (!empty($m[5])) {$exp=$m[5];} - if (!empty($m[6])) {$val=$m[6];} - - // convert to lowercase - if ($this->dom->lowercase) {$tag=strtolower($tag); $key=strtolower($key);} - //elements that do NOT have the specified attribute - if (isset($key[0]) && $key[0]==='!') {$key=substr($key, 1); $no_key=true;} - - $result[] = array($tag, $key, $val, $exp, $no_key); - if (trim($m[7])===',') { + + // Skip NoOps + if ($m[0] === '' || $m[0] === '/' || $m[0] === '//') { continue; } + + // Convert to lowercase + if ($this->dom->lowercase) { + $m[1] = strtolower($m[1]); + } + + // Extract classes + if ($m[3] !== '') { $m[3] = explode('.', $m[3]); } + + /* Extract attributes (pattern based on the pattern above!) + + * [0] - full match + * [1] - attribute name + * [2] - attribute expression + * [3] - attribute value + * [4] - case sensitivity + * + * Note: Attributes can be negated with a "!" prefix to their name + */ + if($m[4] !== '') { + preg_match_all( + "/\[@?(!?[\w:-]+)(?:([!*^$|~]?=)[\"']?(.*?)[\"']?)?(?:\s+?([iIsS])?)?\]/is", + trim($m[4]), + $attributes, + PREG_SET_ORDER + ); + + // Replace element by array + $m[4] = array(); + + foreach($attributes as $att) { + // Skip empty matches + if(trim($att[0]) === '') { continue; } + + $inverted = (isset($att[1][0]) && $att[1][0] === '!'); + $m[4][] = array( + $inverted ? substr($att[1], 1) : $att[1], // Name + (isset($att[2])) ? $att[2] : '', // Expression + (isset($att[3])) ? $att[3] : '', // Value + $inverted, // Inverted Flag + (isset($att[4])) ? strtolower($att[4]) : '', // Case-Sensitivity + ); + } + } + + // Sanitize Separator + if ($m[5] !== '' && trim($m[5]) === '') { // Descendant Separator + $m[5] = ' '; + } else { // Other Separator + $m[5] = trim($m[5]); + } + + // Clear Separator if it's a Selector List + if ($is_list = ($m[5] === ',')) { $m[5] = ''; } + + // Remove full match before adding to results + array_shift($m); + $result[] = $m; + + if ($is_list) { // Selector List $selectors[] = $result; $result = array(); } } - if (count($result)>0) - $selectors[] = $result; + + if (count($result) > 0) { $selectors[] = $result; } return $selectors; } function __get($name) { - if (isset($this->attr[$name])) - { + if (isset($this->attr[$name])) { return $this->convert_text($this->attr[$name]); } - switch ($name) - { + switch ($name) { case 'outertext': return $this->outertext(); case 'innertext': return $this->innertext(); case 'plaintext': return $this->text(); @@ -869,27 +974,28 @@ class simple_html_dom_node function __set($name, $value) { global $debug_object; - if (is_object($debug_object)) {$debug_object->debug_log_entry(1);} + if (is_object($debug_object)) { $debug_object->debug_log_entry(1); } - switch ($name) - { + switch ($name) { case 'outertext': return $this->_[HDOM_INFO_OUTER] = $value; case 'innertext': - if (isset($this->_[HDOM_INFO_TEXT])) return $this->_[HDOM_INFO_TEXT] = $value; + if (isset($this->_[HDOM_INFO_TEXT])) { + return $this->_[HDOM_INFO_TEXT] = $value; + } return $this->_[HDOM_INFO_INNER] = $value; } - if (!isset($this->attr[$name])) - { + + if (!isset($this->attr[$name])) { $this->_[HDOM_INFO_SPACE][] = array(' ', '', ''); $this->_[HDOM_INFO_QUOTE][] = HDOM_QUOTE_DOUBLE; } + $this->attr[$name] = $value; } function __isset($name) { - switch ($name) - { + switch ($name) { case 'outertext': return true; case 'innertext': return true; case 'plaintext': return true; @@ -898,51 +1004,54 @@ class simple_html_dom_node return (array_key_exists($name, $this->attr)) ? true : isset($this->attr[$name]); } - function __unset($name) { - if (isset($this->attr[$name])) - unset($this->attr[$name]); + function __unset($name) + { + if (isset($this->attr[$name])) { unset($this->attr[$name]); } } - // PaperG - Function to convert the text from one character set to another if the two sets are not the same. function convert_text($text) { global $debug_object; - if (is_object($debug_object)) {$debug_object->debug_log_entry(1);} + if (is_object($debug_object)) { $debug_object->debug_log_entry(1); } $converted_text = $text; - $sourceCharset = ""; - $targetCharset = ""; + $sourceCharset = ''; + $targetCharset = ''; - if ($this->dom) - { + if ($this->dom) { $sourceCharset = strtoupper($this->dom->_charset); $targetCharset = strtoupper($this->dom->_target_charset); } - if (is_object($debug_object)) {$debug_object->debug_log(3, "source charset: " . $sourceCharset . " target charaset: " . $targetCharset);} - if (!empty($sourceCharset) && !empty($targetCharset) && (strcasecmp($sourceCharset, $targetCharset) != 0)) - { + if (is_object($debug_object)) { + $debug_object->debug_log(3, + 'source charset: ' + . $sourceCharset + . ' target charaset: ' + . $targetCharset + ); + } + + if (!empty($sourceCharset) + && !empty($targetCharset) + && (strcasecmp($sourceCharset, $targetCharset) != 0)) { // Check if the reported encoding could have been incorrect and the text is actually already UTF-8 - if ((strcasecmp($targetCharset, 'UTF-8') == 0) && ($this->is_utf8($text))) - { + if ((strcasecmp($targetCharset, 'UTF-8') == 0) + && ($this->is_utf8($text))) { $converted_text = $text; - } - else - { + } else { $converted_text = iconv($sourceCharset, $targetCharset, $text); } } // Lets make sure that we don't have that silly BOM issue with any of the utf-8 text we output. - if ($targetCharset == 'UTF-8') - { - if (substr($converted_text, 0, 3) == "\xef\xbb\xbf") - { + if ($targetCharset === 'UTF-8') { + if (substr($converted_text, 0, 3) === "\xef\xbb\xbf") { $converted_text = substr($converted_text, 3); } - if (substr($converted_text, -3) == "\xef\xbb\xbf") - { + + if (substr($converted_text, -3) === "\xef\xbb\xbf") { $converted_text = substr($converted_text, 0, -3); } } @@ -950,57 +1059,33 @@ class simple_html_dom_node return $converted_text; } - /** - * Returns true if $string is valid UTF-8 and false otherwise. - * - * @param mixed $str String to be tested - * @return boolean - */ static function is_utf8($str) { - $c=0; $b=0; - $bits=0; - $len=strlen($str); - for($i=0; $i<$len; $i++) - { - $c=ord($str[$i]); - if($c > 128) - { - if(($c >= 254)) return false; - elseif($c >= 252) $bits=6; - elseif($c >= 248) $bits=5; - elseif($c >= 240) $bits=4; - elseif($c >= 224) $bits=3; - elseif($c >= 192) $bits=2; - else return false; - if(($i+$bits) > $len) return false; - while($bits > 1) - { + $c = 0; $b = 0; + $bits = 0; + $len = strlen($str); + for($i = 0; $i < $len; $i++) { + $c = ord($str[$i]); + if($c > 128) { + if(($c >= 254)) { return false; } + elseif($c >= 252) { $bits = 6; } + elseif($c >= 248) { $bits = 5; } + elseif($c >= 240) { $bits = 4; } + elseif($c >= 224) { $bits = 3; } + elseif($c >= 192) { $bits = 2; } + else { return false; } + if(($i + $bits) > $len) { return false; } + while($bits > 1) { $i++; - $b=ord($str[$i]); - if($b < 128 || $b > 191) return false; + $b = ord($str[$i]); + if($b < 128 || $b > 191) { return false; } $bits--; } } } return true; } - /* - function is_utf8($string) - { - //this is buggy - return (utf8_encode(utf8_decode($string)) == $string); - } - */ - /** - * Function to try a few tricks to determine the displayed size of an img on the page. - * NOTE: This will ONLY work on an IMG tag. Returns FALSE on all other tag types. - * - * @author John Schlick - * @version April 19 2012 - * @return array an array containing the 'height' and 'width' of the image on the page or -1 if we can't figure it out. - */ function get_display_size() { global $debug_object; @@ -1008,383 +1093,393 @@ class simple_html_dom_node $width = -1; $height = -1; - if ($this->tag !== 'img') - { + if ($this->tag !== 'img') { return false; } // See if there is aheight or width attribute in the tag itself. - if (isset($this->attr['width'])) - { + if (isset($this->attr['width'])) { $width = $this->attr['width']; } - if (isset($this->attr['height'])) - { + if (isset($this->attr['height'])) { $height = $this->attr['height']; } // Now look for an inline style. - if (isset($this->attr['style'])) - { + if (isset($this->attr['style'])) { // Thanks to user gnarf from stackoverflow for this regular expression. $attributes = array(); - preg_match_all("/([\w-]+)\s*:\s*([^;]+)\s*;?/", $this->attr['style'], $matches, PREG_SET_ORDER); + + preg_match_all( + '/([\w-]+)\s*:\s*([^;]+)\s*;?/', + $this->attr['style'], + $matches, + PREG_SET_ORDER + ); + foreach ($matches as $match) { - $attributes[$match[1]] = $match[2]; + $attributes[$match[1]] = $match[2]; + } + + // If there is a width in the style attributes: + if (isset($attributes['width']) && $width == -1) { + // check that the last two characters are px (pixels) + if (strtolower(substr($attributes['width'], -2)) === 'px') { + $proposed_width = substr($attributes['width'], 0, -2); + // Now make sure that it's an integer and not something stupid. + if (filter_var($proposed_width, FILTER_VALIDATE_INT)) { + $width = $proposed_width; + } + } + } + + // If there is a width in the style attributes: + if (isset($attributes['height']) && $height == -1) { + // check that the last two characters are px (pixels) + if (strtolower(substr($attributes['height'], -2)) == 'px') { + $proposed_height = substr($attributes['height'], 0, -2); + // Now make sure that it's an integer and not something stupid. + if (filter_var($proposed_height, FILTER_VALIDATE_INT)) { + $height = $proposed_height; + } + } + } + + } + + // Future enhancement: + // Look in the tag to see if there is a class or id specified that has + // a height or width attribute to it. + + // Far future enhancement + // Look at all the parent tags of this image to see if they specify a + // class or id that has an img selector that specifies a height or width + // Note that in this case, the class or id will have the img subselector + // for it to apply to the image. + + // ridiculously far future development + // If the class or id is specified in a SEPARATE css file thats not on + // the page, go get it and do what we were just doing for the ones on + // the page. + + $result = array( + 'height' => $height, + 'width' => $width + ); + + return $result; + } + + function save($filepath = '') + { + $ret = $this->outertext(); + + if ($filepath !== '') { + file_put_contents($filepath, $ret, LOCK_EX); + } + + return $ret; + } + + function addClass($class) + { + if (is_string($class)) { + $class = explode(' ', $class); + } + + if (is_array($class)) { + foreach($class as $c) { + if (isset($this->class)) { + if ($this->hasClass($c)) { + continue; + } else { + $this->class .= ' ' . $c; + } + } else { + $this->class = $c; + } + } + } else { + if (is_object($debug_object)) { + $debug_object->debug_log(2, 'Invalid type: ', gettype($class)); + } + } + } + + function hasClass($class) + { + if (is_string($class)) { + if (isset($this->class)) { + return in_array($class, explode(' ', $this->class), true); + } + } else { + if (is_object($debug_object)) { + $debug_object->debug_log(2, 'Invalid type: ', gettype($class)); } + } + + return false; + } + + function removeClass($class = null) + { + if (!isset($this->class)) { + return; + } + + if (is_null($class)) { + $this->removeAttribute('class'); + return; + } + + if (is_string($class)) { + $class = explode(' ', $class); + } + + if (is_array($class)) { + $class = array_diff(explode(' ', $this->class), $class); + if (empty($class)) { + $this->removeAttribute('class'); + } else { + $this->class = implode(' ', $class); + } + } + } + + function getAllAttributes() + { + return $this->attr; + } + + function getAttribute($name) + { + return $this->__get($name); + } + + function setAttribute($name, $value) + { + $this->__set($name, $value); + } + + function hasAttribute($name) + { + return $this->__isset($name); + } + + function removeAttribute($name) + { + $this->__set($name, null); + } + + function remove() + { + if ($this->parent) { + $this->parent->removeChild($this); + } + } + + function removeChild($node) + { + $nidx = array_search($node, $this->nodes, true); + $cidx = array_search($node, $this->children, true); + $didx = array_search($node, $this->dom->nodes, true); + + if ($nidx !== false && $cidx !== false && $didx !== false) { - // If there is a width in the style attributes: - if (isset($attributes['width']) && $width == -1) - { - // check that the last two characters are px (pixels) - if (strtolower(substr($attributes['width'], -2)) == 'px') - { - $proposed_width = substr($attributes['width'], 0, -2); - // Now make sure that it's an integer and not something stupid. - if (filter_var($proposed_width, FILTER_VALIDATE_INT)) - { - $width = $proposed_width; - } - } + foreach($node->children as $child) { + $node->removeChild($child); } - // If there is a width in the style attributes: - if (isset($attributes['height']) && $height == -1) - { - // check that the last two characters are px (pixels) - if (strtolower(substr($attributes['height'], -2)) == 'px') - { - $proposed_height = substr($attributes['height'], 0, -2); - // Now make sure that it's an integer and not something stupid. - if (filter_var($proposed_height, FILTER_VALIDATE_INT)) - { - $height = $proposed_height; - } + foreach($node->nodes as $entity) { + $enidx = array_search($entity, $node->nodes, true); + $edidx = array_search($entity, $node->dom->nodes, true); + + if ($enidx !== false && $edidx !== false) { + unset($node->nodes[$enidx]); + unset($node->dom->nodes[$edidx]); } } + unset($this->nodes[$nidx]); + unset($this->children[$cidx]); + unset($this->dom->nodes[$didx]); + + $node->clear(); + } + } - // Future enhancement: - // Look in the tag to see if there is a class or id specified that has a height or width attribute to it. + function getElementById($id) + { + return $this->find("#$id", 0); + } - // Far future enhancement - // Look at all the parent tags of this image to see if they specify a class or id that has an img selector that specifies a height or width - // Note that in this case, the class or id will have the img subselector for it to apply to the image. + function getElementsById($id, $idx = null) + { + return $this->find("#$id", $idx); + } - // ridiculously far future development - // If the class or id is specified in a SEPARATE css file thats not on the page, go get it and do what we were just doing for the ones on the page. + function getElementByTagName($name) + { + return $this->find($name, 0); + } - $result = array('height' => $height, - 'width' => $width); - return $result; + function getElementsByTagName($name, $idx = null) + { + return $this->find($name, $idx); + } + + function parentNode() + { + return $this->parent(); + } + + function childNodes($idx = -1) + { + return $this->children($idx); + } + + function firstChild() + { + return $this->first_child(); + } + + function lastChild() + { + return $this->last_child(); + } + + function nextSibling() + { + return $this->next_sibling(); + } + + function previousSibling() + { + return $this->prev_sibling(); } - // camel naming conventions - function getAllAttributes() {return $this->attr;} - function getAttribute($name) {return $this->__get($name);} - function setAttribute($name, $value) {$this->__set($name, $value);} - function hasAttribute($name) {return $this->__isset($name);} - function removeAttribute($name) {$this->__set($name, null);} - function getElementById($id) {return $this->find("#$id", 0);} - function getElementsById($id, $idx=null) {return $this->find("#$id", $idx);} - function getElementByTagName($name) {return $this->find($name, 0);} - function getElementsByTagName($name, $idx=null) {return $this->find($name, $idx);} - function parentNode() {return $this->parent();} - function childNodes($idx=-1) {return $this->children($idx);} - function firstChild() {return $this->first_child();} - function lastChild() {return $this->last_child();} - function nextSibling() {return $this->next_sibling();} - function previousSibling() {return $this->prev_sibling();} - function hasChildNodes() {return $this->has_child();} - function nodeName() {return $this->tag;} - function appendChild($node) {$node->parent($this); return $node;} + function hasChildNodes() + { + return $this->has_child(); + } + + function nodeName() + { + return $this->tag; + } + + function appendChild($node) + { + $node->parent($this); + return $node; + } } -/** - * simple html dom parser - * Paperg - in the find routine: allow us to specify that we want case insensitive testing of the value of the selector. - * Paperg - change $size from protected to public so we can easily access it - * Paperg - added ForceTagsClosed in the constructor which tells us whether we trust the html or not. Default is to NOT trust it. - * - * @package PlaceLocalInclude - */ class simple_html_dom { - /** - * The root node of the document - * - * @var object - */ public $root = null; - - /** - * List of nodes in the current DOM - * - * @var array - */ public $nodes = array(); - - /** - * Callback function to run for each element in the DOM. - * - * @var callable|null - */ public $callback = null; - - /** - * Indicates how tags and attributes are matched - * - * @var bool When set to **true** tags and attributes will be converted to - * lowercase before matching. - */ public $lowercase = false; - - /** - * Original document size - * - * Holds the original document size. - * - * @var int - */ public $original_size; - - /** - * Current document size - * - * Holds the current document size. The document size is determined by the - * string length of ({@see simple_html_dom::$doc}). - * - * _Note_: Using this variable is more efficient than calling `strlen($doc)` - * - * @var int - * */ public $size; - /** - * Current position in the document - * - * @var int - */ protected $pos; - - /** - * The document - * - * @var string - */ protected $doc; - - /** - * Current character - * - * Holds the current character at position {@see simple_html_dom::$pos} in - * the document {@see simple_html_dom::$doc} - * - * _Note_: Using this variable is more efficient than calling `substr($doc, $pos, 1)` - * - * @var string - */ protected $char; protected $cursor; - - /** - * Parent node of the next node detected by the parser - * - * @var object - */ protected $parent; protected $noise = array(); - - /** - * Tokens considered blank in HTML - * - * @var string - */ protected $token_blank = " \t\r\n"; - - /** - * Tokens to identify the equal sign for attributes, stopping either at the - * closing tag ("/" i.e. "") or the end of an opening tag (">" i.e. - * "") - * - * @var string - */ protected $token_equal = ' =/>'; - - /** - * Tokens to identify the end of a tag name. A tag name either ends on the - * ending slash ("/" i.e. "") or whitespace ("\s\r\n\t") - * - * @var string - */ protected $token_slash = " />\r\n\t"; - - /** - * Tokens to identify the end of an attribute - * - * @var string - */ protected $token_attr = ' >'; - // Note that this is referenced by a child node, and so it needs to be public for that node to see this information. public $_charset = ''; public $_target_charset = ''; - /** - * Innertext for
elements - * - * @var string - */ - protected $default_br_text = ""; + protected $default_br_text = ''; - /** - * Suffix for elements - * - * @var string - */ - public $default_span_text = ""; + public $default_span_text = ''; - /** - * Defines a list of self-closing tags (Void elements) according to the HTML - * Specification - * - * _Remarks_: - * - Use `isset()` instead of `in_array()` on array elements to boost - * performance about 30% - * - Sort elements by name for better readability! - * - * @link https://www.w3.org/TR/html HTML Specification - * @link https://www.w3.org/TR/html/syntax.html#void-elements Void elements - */ protected $self_closing_tags = array( - 'area'=>1, - 'base'=>1, - 'br'=>1, - 'col'=>1, - 'embed'=>1, - 'hr'=>1, - 'img'=>1, - 'input'=>1, - 'link'=>1, - 'meta'=>1, - 'param'=>1, - 'source'=>1, - 'track'=>1, - 'wbr'=>1 + 'area' => 1, + 'base' => 1, + 'br' => 1, + 'col' => 1, + 'embed' => 1, + 'hr' => 1, + 'img' => 1, + 'input' => 1, + 'link' => 1, + 'meta' => 1, + 'param' => 1, + 'source' => 1, + 'track' => 1, + 'wbr' => 1 ); - - /** - * Defines a list of tags which - if closed - close all optional closing - * elements within if they haven't been closed yet. (So, an element where - * neither opening nor closing tag is omissible consistently closes every - * optional closing element within) - * - * _Remarks_: - * - Use `isset()` instead of `in_array()` on array elements to boost - * performance about 30% - * - Sort elements by name for better readability! - */ protected $block_tags = array( - 'body'=>1, - 'div'=>1, - 'form'=>1, - 'root'=>1, - 'span'=>1, - 'table'=>1 + 'body' => 1, + 'div' => 1, + 'form' => 1, + 'root' => 1, + 'span' => 1, + 'table' => 1 ); - - /** - * Defines elements whose end tag is omissible. - * - * * key = Name of an element whose end tag is omissible. - * * value = Names of elements whose end tag is omissible, that are closed - * by the current element. - * - * _Remarks_: - * - Use `isset()` instead of `in_array()` on array elements to boost - * performance about 30% - * - Sort elements by name for better readability! - * - * **Example** - * - * An `li` element’s end tag may be omitted if the `li` element is immediately - * followed by another `li` element. To do that, add following element to the - * array: - * - * ```php - * 'li' => array('li'), - * ``` - * - * With this, the following two examples are considered equal. Note that the - * second example is missing the closing tags on `li` elements. - * - * ```html - *
  • First Item
  • Second Item
- * ``` - * - *
  • First Item
  • Second Item
- * - * ```html - *
  • First Item
  • Second Item
- * ``` - * - *
  • First Item
  • Second Item
- * - * @var array A two-dimensional array where the key is the name of an - * element whose end tag is omissible and the value is an array of elements - * whose end tag is omissible, that are closed by the current element. - * - * @link https://www.w3.org/TR/html/syntax.html#optional-tags Optional tags - * - * @todo The implementation of optional closing tags doesn't work in all cases - * because it only consideres elements who close other optional closing - * tags, not taking into account that some (non-blocking) tags should close - * these optional closing tags. For example, the end tag for "p" is omissible - * and can be closed by an "address" element, whose end tag is NOT omissible. - * Currently a "p" element without closing tag stops at the next "p" element - * or blocking tag, even if it contains other elements. - * - * @todo Known sourceforge issue #2977341 - * B tags that are not closed cause us to return everything to the end of - * the document. - */ protected $optional_closing_tags = array( - 'b'=>array('b'=>1), // Not optional, see https://www.w3.org/TR/html/textlevel-semantics.html#the-b-element - 'dd'=>array('dd'=>1, 'dt'=>1), - 'dl'=>array('dd'=>1, 'dt'=>1), // Not optional, see https://www.w3.org/TR/html/grouping-content.html#the-dl-element - 'dt'=>array('dd'=>1, 'dt'=>1), - 'li'=>array('li'=>1), - 'optgroup'=>array('optgroup'=>1, 'option'=>1), - 'option'=>array('optgroup'=>1, 'option'=>1), - 'p'=>array('p'=>1), - 'rp'=>array('rp'=>1, 'rt'=>1), - 'rt'=>array('rp'=>1, 'rt'=>1), - 'td'=>array('td'=>1, 'th'=>1), - 'th'=>array('td'=>1, 'th'=>1), - 'tr'=>array('td'=>1, 'th'=>1, 'tr'=>1), + // Not optional, see + // https://www.w3.org/TR/html/textlevel-semantics.html#the-b-element + 'b' => array('b' => 1), + 'dd' => array('dd' => 1, 'dt' => 1), + // Not optional, see + // https://www.w3.org/TR/html/grouping-content.html#the-dl-element + 'dl' => array('dd' => 1, 'dt' => 1), + 'dt' => array('dd' => 1, 'dt' => 1), + 'li' => array('li' => 1), + 'optgroup' => array('optgroup' => 1, 'option' => 1), + 'option' => array('optgroup' => 1, 'option' => 1), + 'p' => array('p' => 1), + 'rp' => array('rp' => 1, 'rt' => 1), + 'rt' => array('rp' => 1, 'rt' => 1), + 'td' => array('td' => 1, 'th' => 1), + 'th' => array('td' => 1, 'th' => 1), + 'tr' => array('td' => 1, 'th' => 1, 'tr' => 1), ); - function __construct($str=null, $lowercase=true, $forceTagsClosed=true, $target_charset=DEFAULT_TARGET_CHARSET, $stripRN=true, $defaultBRText=DEFAULT_BR_TEXT, $defaultSpanText=DEFAULT_SPAN_TEXT, $options=0) + function __construct( + $str = null, + $lowercase = true, + $forceTagsClosed = true, + $target_charset = DEFAULT_TARGET_CHARSET, + $stripRN = true, + $defaultBRText = DEFAULT_BR_TEXT, + $defaultSpanText = DEFAULT_SPAN_TEXT, + $options = 0) { - if ($str) - { - if (preg_match("/^http:\/\//i",$str) || is_file($str)) - { + if ($str) { + if (preg_match('/^http:\/\//i', $str) || is_file($str)) { $this->load_file($str); - } - else - { - $this->load($str, $lowercase, $stripRN, $defaultBRText, $defaultSpanText, $options); + } else { + $this->load( + $str, + $lowercase, + $stripRN, + $defaultBRText, + $defaultSpanText, + $options + ); } } - // Forcing tags to be closed implies that we don't trust the html, but it can lead to parsing errors if we SHOULD trust the html. + // Forcing tags to be closed implies that we don't trust the html, but + // it can lead to parsing errors if we SHOULD trust the html. if (!$forceTagsClosed) { - $this->optional_closing_array=array(); + $this->optional_closing_array = array(); } + $this->_target_charset = $target_charset; } @@ -1393,8 +1488,13 @@ class simple_html_dom $this->clear(); } - // load html from string - function load($str, $lowercase=true, $stripRN=true, $defaultBRText=DEFAULT_BR_TEXT, $defaultSpanText=DEFAULT_SPAN_TEXT, $options=0) + function load( + $str, + $lowercase = true, + $stripRN = true, + $defaultBRText = DEFAULT_BR_TEXT, + $defaultSpanText = DEFAULT_SPAN_TEXT, + $options = 0) { global $debug_object; @@ -1409,8 +1509,8 @@ class simple_html_dom // strip out the \r \n's if we are told to. if ($stripRN) { - $this->doc = str_replace("\r", " ", $this->doc); - $this->doc = str_replace("\n", " ", $this->doc); + $this->doc = str_replace("\r", ' ', $this->doc); + $this->doc = str_replace("\n", ' ', $this->doc); // set the length of content since we have changed it. $this->size = strlen($this->doc); @@ -1440,83 +1540,89 @@ class simple_html_dom // make load function chainable return $this; - } - // load html from file function load_file() { $args = func_get_args(); - if($doc = call_user_func_array('file_get_contents', $args) !== false) { + if(($doc = call_user_func_array('file_get_contents', $args)) !== false) { $this->load($doc, true); } else { return false; } } - /** - * Set the callback function - * - * @param callable $function_name Callback function to run for each element - * in the DOM. - * @return void - */ function set_callback($function_name) { $this->callback = $function_name; } - /** - * Remove callback function - * - * @return void - */ function remove_callback() { $this->callback = null; } - // save dom as string - function save($filepath='') + function save($filepath = '') { $ret = $this->root->innertext(); - if ($filepath!=='') file_put_contents($filepath, $ret, LOCK_EX); + if ($filepath !== '') { file_put_contents($filepath, $ret, LOCK_EX); } return $ret; } - // find dom node by css selector - // Paperg - allow us to specify that we want case insensitive testing of the value of the selector. - function find($selector, $idx=null, $lowercase=false) + function find($selector, $idx = null, $lowercase = false) { return $this->root->find($selector, $idx, $lowercase); } - // clean up memory due to php5 circular references memory leak... function clear() { - foreach ($this->nodes as $n) {$n->clear(); $n = null;} - // This add next line is documented in the sourceforge repository. 2977248 as a fix for ongoing memory leaks that occur even with the use of clear. - if (isset($this->children)) foreach ($this->children as $n) {$n->clear(); $n = null;} - if (isset($this->parent)) {$this->parent->clear(); unset($this->parent);} - if (isset($this->root)) {$this->root->clear(); unset($this->root);} + if (isset($this->nodes)) { + foreach ($this->nodes as $n) { + $n->clear(); + $n = null; + } + } + + // This add next line is documented in the sourceforge repository. + // 2977248 as a fix for ongoing memory leaks that occur even with the + // use of clear. + if (isset($this->children)) { + foreach ($this->children as $n) { + $n->clear(); + $n = null; + } + } + + if (isset($this->parent)) { + $this->parent->clear(); + unset($this->parent); + } + + if (isset($this->root)) { + $this->root->clear(); + unset($this->root); + } + unset($this->doc); unset($this->noise); } - function dump($show_attr=true) + function dump($show_attr = true) { $this->root->dump($show_attr); } - // prepare HTML data and init everything - protected function prepare($str, $lowercase=true, $defaultBRText=DEFAULT_BR_TEXT, $defaultSpanText=DEFAULT_SPAN_TEXT) + protected function prepare( + $str, $lowercase = true, + $defaultBRText = DEFAULT_BR_TEXT, + $defaultSpanText = DEFAULT_SPAN_TEXT) { $this->clear(); $this->doc = trim($str); $this->size = strlen($this->doc); - $this->original_size = $this->size; // Save the original size of the html that we got in. It might be useful to someone. + $this->original_size = $this->size; // original size of the html $this->pos = 0; $this->cursor = 1; $this->noise = array(); @@ -1529,21 +1635,15 @@ class simple_html_dom $this->root->_[HDOM_INFO_BEGIN] = -1; $this->root->nodetype = HDOM_TYPE_ROOT; $this->parent = $this->root; - if ($this->size>0) $this->char = $this->doc[0]; + if ($this->size > 0) { $this->char = $this->doc[0]; } } - /** - * Parse HTML content - * - * @return bool True on success - */ protected function parse() { while (true) { // Read next tag if there is no text between current position and the // next opening tag. - if (($s = $this->copy_until_char('<'))==='') - { + if (($s = $this->copy_until_char('<')) === '') { if($this->read_tag()) { continue; } else { @@ -1559,174 +1659,241 @@ class simple_html_dom } } - // PAPERG - dkchou - added this to try to identify the character set of the page we have just parsed so we know better how to spit it out later. - // NOTE: IF you provide a routine called get_last_retrieve_url_contents_content_type which returns the CURLINFO_CONTENT_TYPE from the last curl_exec - // (or the content_type header from the last transfer), we will parse THAT, and if a charset is specified, we will use it over any other mechanism. protected function parse_charset() { global $debug_object; $charset = null; - if (function_exists('get_last_retrieve_url_contents_content_type')) - { + if (function_exists('get_last_retrieve_url_contents_content_type')) { $contentTypeHeader = get_last_retrieve_url_contents_content_type(); $success = preg_match('/charset=(.+)/', $contentTypeHeader, $matches); - if ($success) - { + if ($success) { $charset = $matches[1]; - if (is_object($debug_object)) {$debug_object->debug_log(2, 'header content-type found charset of: ' . $charset);} + if (is_object($debug_object)) { + $debug_object->debug_log(2, + 'header content-type found charset of: ' + . $charset + ); + } } - } - if (empty($charset)) - { - $el = $this->root->find('meta[http-equiv=Content-Type]',0, true); - if (!empty($el)) - { + if (empty($charset)) { + // https://www.w3.org/TR/html/document-metadata.html#statedef-http-equiv-content-type + $el = $this->root->find('meta[http-equiv=Content-Type]', 0, true); + + if (!empty($el)) { $fullvalue = $el->content; - if (is_object($debug_object)) {$debug_object->debug_log(2, 'meta content-type tag found' . $fullvalue);} + if (is_object($debug_object)) { + $debug_object->debug_log(2, + 'meta content-type tag found' + . $fullvalue + ); + } - if (!empty($fullvalue)) - { - $success = preg_match('/charset=(.+)/i', $fullvalue, $matches); - if ($success) - { + if (!empty($fullvalue)) { + $success = preg_match( + '/charset=(.+)/i', + $fullvalue, + $matches + ); + + if ($success) { $charset = $matches[1]; - } - else - { - // If there is a meta tag, and they don't specify the character set, research says that it's typically ISO-8859-1 - if (is_object($debug_object)) {$debug_object->debug_log(2, 'meta content-type tag couldn\'t be parsed. using iso-8859 default.');} + } else { + // If there is a meta tag, and they don't specify the + // character set, research says that it's typically + // ISO-8859-1 + if (is_object($debug_object)) { + $debug_object->debug_log(2, + 'meta content-type tag couldn\'t be parsed. using iso-8859 default.' + ); + } + $charset = 'ISO-8859-1'; } } } } - // If we couldn't find a charset above, then lets try to detect one based on the text we got... - if (empty($charset)) - { - // Use this in case mb_detect_charset isn't installed/loaded on this machine. - $charset = false; - if (function_exists('mb_detect_encoding')) - { - // Have php try to detect the encoding from the text given to us. - $charset = mb_detect_encoding($this->doc . "ascii", $encoding_list = array( "UTF-8", "CP1252" ) ); - if (is_object($debug_object)) {$debug_object->debug_log(2, 'mb_detect found: ' . $charset);} + if (empty($charset)) { + // https://www.w3.org/TR/html/document-metadata.html#character-encoding-declaration + if ($meta = $this->root->find('meta[charset]', 0)) { + $charset = $meta->charset; + if (is_object($debug_object)) { + $debug_object->debug_log(2, 'meta charset: ' . $charset); + } + } + } + + if (empty($charset)) { + // Try to guess the charset based on the content + // Requires Multibyte String (mbstring) support (optional) + if (function_exists('mb_detect_encoding')) { + /** + * mb_detect_encoding() is not intended to distinguish between + * charsets, especially single-byte charsets. Its primary + * purpose is to detect which multibyte encoding is in use, + * i.e. UTF-8, UTF-16, shift-JIS, etc. + * + * -- https://bugs.php.net/bug.php?id=38138 + * + * Adding both CP1251/ISO-8859-5 and CP1252/ISO-8859-1 will + * always result in CP1251/ISO-8859-5 and vice versa. + * + * Thus, only detect if it's either UTF-8 or CP1252/ISO-8859-1 + * to stay compatible. + */ + $encoding = mb_detect_encoding( + $this->doc, + array( 'UTF-8', 'CP1252', 'ISO-8859-1' ) + ); + + if ($encoding === 'CP1252' || $encoding === 'ISO-8859-1') { + // Due to a limitation of mb_detect_encoding + // 'CP1251'/'ISO-8859-5' will be detected as + // 'CP1252'/'ISO-8859-1'. This will cause iconv to fail, in + // which case we can simply assume it is the other charset. + if (!@iconv('CP1252', 'UTF-8', $this->doc)) { + $encoding = 'CP1251'; + } + } + + if ($encoding !== false) { + $charset = $encoding; + if (is_object($debug_object)) { + $debug_object->debug_log(2, 'mb_detect: ' . $charset); + } + } } + } - // and if this doesn't work... then we need to just wrongheadedly assume it's UTF-8 so that we can move on - cause this will usually give us most of what we need... - if ($charset === false) - { - if (is_object($debug_object)) {$debug_object->debug_log(2, 'since mb_detect failed - using default of utf-8');} - $charset = 'UTF-8'; + if (empty($charset)) { + // Assume it's UTF-8 as it is the most likely charset to be used + $charset = 'UTF-8'; + if (is_object($debug_object)) { + $debug_object->debug_log(2, 'No match found, assume ' . $charset); } } - // Since CP1252 is a superset, if we get one of it's subsets, we want it instead. - if ((strtolower($charset) == strtolower('ISO-8859-1')) || (strtolower($charset) == strtolower('Latin1')) || (strtolower($charset) == strtolower('Latin-1'))) - { - if (is_object($debug_object)) {$debug_object->debug_log(2, 'replacing ' . $charset . ' with CP1252 as its a superset');} + // Since CP1252 is a superset, if we get one of it's subsets, we want + // it instead. + if ((strtolower($charset) == 'iso-8859-1') + || (strtolower($charset) == 'latin1') + || (strtolower($charset) == 'latin-1')) { $charset = 'CP1252'; + if (is_object($debug_object)) { + $debug_object->debug_log(2, + 'replacing ' . $charset . ' with CP1252 as its a superset' + ); + } } - if (is_object($debug_object)) {$debug_object->debug_log(1, 'EXIT - ' . $charset);} + if (is_object($debug_object)) { + $debug_object->debug_log(1, 'EXIT - ' . $charset); + } return $this->_charset = $charset; } - /** - * Parse tag from current document position. - * - * @return bool True if a tag was found, false otherwise - */ protected function read_tag() { // Set end position if no further tags found - if ($this->char!=='<') - { + if ($this->char !== '<') { $this->root->_[HDOM_INFO_END] = $this->cursor; return false; } + $begin_tag_pos = $this->pos; - $this->char = (++$this->pos<$this->size) ? $this->doc[$this->pos] : null; // next + $this->char = (++$this->pos < $this->size) ? $this->doc[$this->pos] : null; // next // end tag - if ($this->char==='/') - { - $this->char = (++$this->pos<$this->size) ? $this->doc[$this->pos] : null; // next + if ($this->char === '/') { + $this->char = (++$this->pos < $this->size) ? $this->doc[$this->pos] : null; // next // Skip whitespace in end tags (i.e. in "") $this->skip($this->token_blank); $tag = $this->copy_until_char('>'); // Skip attributes in end tags - if (($pos = strpos($tag, ' '))!==false) + if (($pos = strpos($tag, ' ')) !== false) { $tag = substr($tag, 0, $pos); + } $parent_lower = strtolower($this->parent->tag); $tag_lower = strtolower($tag); // The end tag is supposed to close the parent tag. Handle situations // when it doesn't - if ($parent_lower!==$tag_lower) - { + if ($parent_lower !== $tag_lower) { // Parent tag does not have to be closed necessarily (optional closing tag) // Current tag is a block tag, so it may close an ancestor - if (isset($this->optional_closing_tags[$parent_lower]) && isset($this->block_tags[$tag_lower])) - { + if (isset($this->optional_closing_tags[$parent_lower]) + && isset($this->block_tags[$tag_lower])) { + $this->parent->_[HDOM_INFO_END] = 0; $org_parent = $this->parent; // Traverse ancestors to find a matching opening tag // Stop at root node - while (($this->parent->parent) && strtolower($this->parent->tag)!==$tag_lower) + while (($this->parent->parent) + && strtolower($this->parent->tag) !== $tag_lower + ){ $this->parent = $this->parent->parent; + } // If we don't have a match add current tag as text node - if (strtolower($this->parent->tag)!==$tag_lower) { + if (strtolower($this->parent->tag) !== $tag_lower) { $this->parent = $org_parent; // restore origonal parent - if ($this->parent->parent) $this->parent = $this->parent->parent; + + if ($this->parent->parent) { + $this->parent = $this->parent->parent; + } + $this->parent->_[HDOM_INFO_END] = $this->cursor; return $this->as_text_node($tag); } - } - // Grandparent exists and current tag is a block tag, so our parent doesn't have an end tag - else if (($this->parent->parent) && isset($this->block_tags[$tag_lower])) - { + } elseif (($this->parent->parent) + && isset($this->block_tags[$tag_lower]) + ) { + // Grandparent exists and current tag is a block tag, so our + // parent doesn't have an end tag $this->parent->_[HDOM_INFO_END] = 0; // No end tag $org_parent = $this->parent; // Traverse ancestors to find a matching opening tag // Stop at root node - while (($this->parent->parent) && strtolower($this->parent->tag)!==$tag_lower) + while (($this->parent->parent) + && strtolower($this->parent->tag) !== $tag_lower + ) { $this->parent = $this->parent->parent; + } // If we don't have a match add current tag as text node - if (strtolower($this->parent->tag)!==$tag_lower) - { + if (strtolower($this->parent->tag) !== $tag_lower) { $this->parent = $org_parent; // restore origonal parent $this->parent->_[HDOM_INFO_END] = $this->cursor; return $this->as_text_node($tag); } - } - // Grandparent exists and current tag closes it - else if (($this->parent->parent) && strtolower($this->parent->parent->tag)===$tag_lower) - { + } elseif (($this->parent->parent) + && strtolower($this->parent->parent->tag) === $tag_lower + ) { // Grandparent exists and current tag closes it $this->parent->_[HDOM_INFO_END] = 0; $this->parent = $this->parent->parent; - } - else // Random tag, add as text node + } else { // Random tag, add as text node return $this->as_text_node($tag); + } } // Set end position of parent tag to current cursor position $this->parent->_[HDOM_INFO_END] = $this->cursor; - if ($this->parent->parent) $this->parent = $this->parent->parent; - $this->char = (++$this->pos<$this->size) ? $this->doc[$this->pos] : null; // next + if ($this->parent->parent) { + $this->parent = $this->parent->parent; + } + + $this->char = (++$this->pos < $this->size) ? $this->doc[$this->pos] : null; // next return true; } @@ -1741,25 +1908,27 @@ class simple_html_dom // // // - if (isset($tag[0]) && $tag[0]==='!') { + if (isset($tag[0]) && $tag[0] === '!') { $node->_[HDOM_INFO_TEXT] = '<' . $tag . $this->copy_until_char('>'); - if (isset($tag[2]) && $tag[1]==='-' && $tag[2]==='-') { // Comment ("