From aaa1f3d07a886852647ba03aa7217cd7066b5c5e Mon Sep 17 00:00:00 2001 From: Abraham Raji Date: Tue, 30 Jun 2020 07:04:39 +0200 Subject: Import ruby-typhoeus_1.4.0.orig.tar.gz [dgit import orig ruby-typhoeus_1.4.0.orig.tar.gz] --- .gitignore | 8 + .rspec | 4 + .travis.yml | 26 + CHANGELOG.md | 406 ++++++++++++++ CONTRIBUTING.md | 20 + Gemfile | 32 ++ Guardfile | 9 + LICENSE | 22 + README.md | 584 +++++++++++++++++++++ Rakefile | 38 ++ UPGRADE.md | 55 ++ lib/rack/typhoeus.rb | 1 + lib/rack/typhoeus/middleware/params_decoder.rb | 57 ++ .../typhoeus/middleware/params_decoder/helper.rb | 76 +++ lib/typhoeus.rb | 143 +++++ lib/typhoeus/adapters/faraday.rb | 180 +++++++ lib/typhoeus/cache/dalli.rb | 28 + lib/typhoeus/cache/rails.rb | 28 + lib/typhoeus/cache/redis.rb | 35 ++ lib/typhoeus/config.rb | 69 +++ lib/typhoeus/easy_factory.rb | 180 +++++++ lib/typhoeus/errors.rb | 9 + lib/typhoeus/errors/no_stub.rb | 12 + lib/typhoeus/errors/typhoeus_error.rb | 8 + lib/typhoeus/expectation.rb | 217 ++++++++ lib/typhoeus/hydra.rb | 95 ++++ lib/typhoeus/hydra/addable.rb | 23 + lib/typhoeus/hydra/before.rb | 31 ++ lib/typhoeus/hydra/block_connection.rb | 35 ++ lib/typhoeus/hydra/cacheable.rb | 15 + lib/typhoeus/hydra/memoizable.rb | 56 ++ lib/typhoeus/hydra/queueable.rb | 83 +++ lib/typhoeus/hydra/runnable.rb | 19 + lib/typhoeus/hydra/stubbable.rb | 28 + lib/typhoeus/pool.rb | 70 +++ lib/typhoeus/railtie.rb | 12 + lib/typhoeus/request.rb | 221 ++++++++ lib/typhoeus/request/actions.rb | 125 +++++ lib/typhoeus/request/before.rb | 30 ++ lib/typhoeus/request/block_connection.rb | 52 ++ lib/typhoeus/request/cacheable.rb | 38 ++ lib/typhoeus/request/callbacks.rb | 151 ++++++ lib/typhoeus/request/marshal.rb | 22 + lib/typhoeus/request/memoizable.rb | 38 ++ lib/typhoeus/request/operations.rb | 40 ++ lib/typhoeus/request/responseable.rb | 29 + lib/typhoeus/request/streamable.rb | 34 ++ lib/typhoeus/request/stubbable.rb | 30 ++ lib/typhoeus/response.rb | 68 +++ lib/typhoeus/response/cacheable.rb | 14 + lib/typhoeus/response/header.rb | 105 ++++ lib/typhoeus/response/informations.rb | 248 +++++++++ lib/typhoeus/response/status.rb | 106 ++++ lib/typhoeus/version.rb | 5 + perf/profile.rb | 14 + perf/vs_nethttp.rb | 64 +++ .../middleware/params_decoder/helper_spec.rb | 156 ++++++ .../typhoeus/middleware/params_decoder_spec.rb | 31 ++ spec/spec_helper.rb | 29 + spec/support/localhost_server.rb | 94 ++++ spec/support/memory_cache.rb | 15 + spec/support/server.rb | 116 ++++ spec/typhoeus/adapters/faraday_spec.rb | 339 ++++++++++++ spec/typhoeus/cache/dalli_spec.rb | 41 ++ spec/typhoeus/cache/redis_spec.rb | 41 ++ spec/typhoeus/config_spec.rb | 15 + spec/typhoeus/easy_factory_spec.rb | 143 +++++ spec/typhoeus/errors/no_stub_spec.rb | 13 + spec/typhoeus/expectation_spec.rb | 280 ++++++++++ spec/typhoeus/hydra/addable_spec.rb | 22 + spec/typhoeus/hydra/before_spec.rb | 98 ++++ spec/typhoeus/hydra/block_connection_spec.rb | 18 + spec/typhoeus/hydra/cacheable_spec.rb | 88 ++++ spec/typhoeus/hydra/memoizable_spec.rb | 53 ++ spec/typhoeus/hydra/queueable_spec.rb | 98 ++++ spec/typhoeus/hydra/runnable_spec.rb | 137 +++++ spec/typhoeus/hydra/stubbable_spec.rb | 48 ++ spec/typhoeus/hydra_spec.rb | 22 + spec/typhoeus/pool_spec.rb | 137 +++++ spec/typhoeus/request/actions_spec.rb | 19 + spec/typhoeus/request/before_spec.rb | 93 ++++ spec/typhoeus/request/block_connection_spec.rb | 75 +++ spec/typhoeus/request/cacheable_spec.rb | 94 ++++ spec/typhoeus/request/callbacks_spec.rb | 91 ++++ spec/typhoeus/request/marshal_spec.rb | 60 +++ spec/typhoeus/request/memoizable_spec.rb | 34 ++ spec/typhoeus/request/operations_spec.rb | 101 ++++ spec/typhoeus/request/responseable_spec.rb | 13 + spec/typhoeus/request/stubbable_spec.rb | 45 ++ spec/typhoeus/request_spec.rb | 232 ++++++++ spec/typhoeus/response/header_spec.rb | 147 ++++++ spec/typhoeus/response/informations_spec.rb | 283 ++++++++++ spec/typhoeus/response/status_spec.rb | 256 +++++++++ spec/typhoeus/response_spec.rb | 100 ++++ spec/typhoeus_spec.rb | 105 ++++ typhoeus.gemspec | 25 + 96 files changed, 7955 insertions(+) create mode 100644 .gitignore create mode 100644 .rspec create mode 100644 .travis.yml create mode 100644 CHANGELOG.md create mode 100644 CONTRIBUTING.md create mode 100644 Gemfile create mode 100644 Guardfile create mode 100644 LICENSE create mode 100644 README.md create mode 100644 Rakefile create mode 100644 UPGRADE.md create mode 100644 lib/rack/typhoeus.rb create mode 100644 lib/rack/typhoeus/middleware/params_decoder.rb create mode 100644 lib/rack/typhoeus/middleware/params_decoder/helper.rb create mode 100644 lib/typhoeus.rb create mode 100644 lib/typhoeus/adapters/faraday.rb create mode 100644 lib/typhoeus/cache/dalli.rb create mode 100644 lib/typhoeus/cache/rails.rb create mode 100644 lib/typhoeus/cache/redis.rb create mode 100644 lib/typhoeus/config.rb create mode 100644 lib/typhoeus/easy_factory.rb create mode 100644 lib/typhoeus/errors.rb create mode 100644 lib/typhoeus/errors/no_stub.rb create mode 100644 lib/typhoeus/errors/typhoeus_error.rb create mode 100644 lib/typhoeus/expectation.rb create mode 100644 lib/typhoeus/hydra.rb create mode 100644 lib/typhoeus/hydra/addable.rb create mode 100644 lib/typhoeus/hydra/before.rb create mode 100644 lib/typhoeus/hydra/block_connection.rb create mode 100644 lib/typhoeus/hydra/cacheable.rb create mode 100644 lib/typhoeus/hydra/memoizable.rb create mode 100644 lib/typhoeus/hydra/queueable.rb create mode 100644 lib/typhoeus/hydra/runnable.rb create mode 100644 lib/typhoeus/hydra/stubbable.rb create mode 100644 lib/typhoeus/pool.rb create mode 100644 lib/typhoeus/railtie.rb create mode 100644 lib/typhoeus/request.rb create mode 100644 lib/typhoeus/request/actions.rb create mode 100644 lib/typhoeus/request/before.rb create mode 100644 lib/typhoeus/request/block_connection.rb create mode 100644 lib/typhoeus/request/cacheable.rb create mode 100644 lib/typhoeus/request/callbacks.rb create mode 100644 lib/typhoeus/request/marshal.rb create mode 100644 lib/typhoeus/request/memoizable.rb create mode 100644 lib/typhoeus/request/operations.rb create mode 100644 lib/typhoeus/request/responseable.rb create mode 100644 lib/typhoeus/request/streamable.rb create mode 100644 lib/typhoeus/request/stubbable.rb create mode 100644 lib/typhoeus/response.rb create mode 100644 lib/typhoeus/response/cacheable.rb create mode 100644 lib/typhoeus/response/header.rb create mode 100644 lib/typhoeus/response/informations.rb create mode 100644 lib/typhoeus/response/status.rb create mode 100644 lib/typhoeus/version.rb create mode 100644 perf/profile.rb create mode 100644 perf/vs_nethttp.rb create mode 100644 spec/rack/typhoeus/middleware/params_decoder/helper_spec.rb create mode 100644 spec/rack/typhoeus/middleware/params_decoder_spec.rb create mode 100644 spec/spec_helper.rb create mode 100644 spec/support/localhost_server.rb create mode 100644 spec/support/memory_cache.rb create mode 100644 spec/support/server.rb create mode 100644 spec/typhoeus/adapters/faraday_spec.rb create mode 100644 spec/typhoeus/cache/dalli_spec.rb create mode 100644 spec/typhoeus/cache/redis_spec.rb create mode 100644 spec/typhoeus/config_spec.rb create mode 100644 spec/typhoeus/easy_factory_spec.rb create mode 100644 spec/typhoeus/errors/no_stub_spec.rb create mode 100644 spec/typhoeus/expectation_spec.rb create mode 100644 spec/typhoeus/hydra/addable_spec.rb create mode 100644 spec/typhoeus/hydra/before_spec.rb create mode 100644 spec/typhoeus/hydra/block_connection_spec.rb create mode 100644 spec/typhoeus/hydra/cacheable_spec.rb create mode 100644 spec/typhoeus/hydra/memoizable_spec.rb create mode 100644 spec/typhoeus/hydra/queueable_spec.rb create mode 100644 spec/typhoeus/hydra/runnable_spec.rb create mode 100644 spec/typhoeus/hydra/stubbable_spec.rb create mode 100644 spec/typhoeus/hydra_spec.rb create mode 100644 spec/typhoeus/pool_spec.rb create mode 100644 spec/typhoeus/request/actions_spec.rb create mode 100644 spec/typhoeus/request/before_spec.rb create mode 100644 spec/typhoeus/request/block_connection_spec.rb create mode 100644 spec/typhoeus/request/cacheable_spec.rb create mode 100644 spec/typhoeus/request/callbacks_spec.rb create mode 100644 spec/typhoeus/request/marshal_spec.rb create mode 100644 spec/typhoeus/request/memoizable_spec.rb create mode 100644 spec/typhoeus/request/operations_spec.rb create mode 100644 spec/typhoeus/request/responseable_spec.rb create mode 100644 spec/typhoeus/request/stubbable_spec.rb create mode 100644 spec/typhoeus/request_spec.rb create mode 100644 spec/typhoeus/response/header_spec.rb create mode 100644 spec/typhoeus/response/informations_spec.rb create mode 100644 spec/typhoeus/response/status_spec.rb create mode 100644 spec/typhoeus/response_spec.rb create mode 100644 spec/typhoeus_spec.rb create mode 100644 typhoeus.gemspec diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..a414378 --- /dev/null +++ b/.gitignore @@ -0,0 +1,8 @@ +*.gem +Gemfile.lock +doc/ +.yardoc +.rvmrc +coverage +check.sh +tags diff --git a/.rspec b/.rspec new file mode 100644 index 0000000..838fd57 --- /dev/null +++ b/.rspec @@ -0,0 +1,4 @@ +--tty +--color +--format documentation +--backtrace diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..dc0c9df --- /dev/null +++ b/.travis.yml @@ -0,0 +1,26 @@ +language: ruby +script: "bundle exec rake" +rvm: + - 1.9.3 + - 2.0.0 + - 2.1.10 + - 2.2.10 + - 2.3.8 + - 2.4.7 + - 2.5.6 + - 2.6.4 + - ruby-head + - jruby-head + - jruby-18mode + - jruby-19mode +matrix: + fast_finish: true + allow_failures: + - rvm: ruby-head + - rvm: jruby-head + - rvm: ree + include: + - rvm: 1.8.7 + dist: precise + - rvm: 1.9.2 + dist: trusty diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..223e4bc --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,406 @@ +# Changelog + +## Master + +[Full Changelog](http://github.com/typhoeus/typhoeus/compare/v1.4.0...master) + +## 1.4.0 + +[Full Changelog](http://github.com/typhoeus/typhoeus/compare/v1.1.2...v1.4.0) + +#### 1 feature +- Faraday adapter exceptions namespace compatibility with Faraday v1 ([@iMacTia](https://github.com/iMacTia) in [#616](https://github.com/typhoeus/typhoeus/pull/616)) + +#### 3 Others +- Yard warning fixes ([@olleolleolle](https://github.com/olleolleolle) in [#622](https://github.com/typhoeus/typhoeus/pull/622)) +- Add more Ruby versions in CI matrix ([@olleolleolle](https://github.com/olleolleolle) in [#623](https://github.com/typhoeus/typhoeus/pull/623)) +- Use of argument passed in function instead of `attr_reader` ([@v-kolesnikov](https://github.com/v-kolesnikov) in [#625](https://github.com/typhoeus/typhoeus/pull/625)) + +## 1.1.2 + +[Full Changelog](http://github.com/typhoeus/typhoeus/compare/v1.1.1...v1.1.2) + +## 1.1.1 + +[Full Changelog](http://github.com/typhoeus/typhoeus/compare/v1.1.0...v1.1.1) + +## 1.1.0 + +[Full Changelog](http://github.com/typhoeus/typhoeus/compare/v1.0.2...v1.1.0) + +* Unless specified `Expect` header is set to be empty to avoid `100 continue` + to be set when using `PUT` +* Add global config option `Typhoeus::Config.proxy` + +## 1.0.2 + +[Full Changelog](http://github.com/typhoeus/typhoeus/compare/v1.0.1...v1.0.2) + +## 1.0.1 + +[Full Changelog](http://github.com/typhoeus/typhoeus/compare/v1.0.0...v1.0.1) + +## 1.0.0 + +[Full Changelog](http://github.com/typhoeus/typhoeus/compare/v0.8.0...v1.0.0) + +## 0.8.0 + +[Full Changelog](http://github.com/typhoeus/typhoeus/compare/v0.7.3...v0.8.0) + +* `EasyFactory`: Reduced object allocations and method calls during deprecated + option handling and option sanitization. + ([Tasos Laskos](https://github.com/zapotek)) +* `Response` ([Tasos Laskos](https://github.com/zapotek)) + * `Header` + * `#process_pair`: Halved `#set_value` calls. + * `#set_value`: Minimized `Hash` accesses. + * `#parse`: Use `String#start_with?` instead of `Regexp` match. + * `#process_line`: Optimized key/value sanitization. + * `Status` + * `#timed_out?`: Only return `true` when `#return_code` is `operation_timedout`. + +## 0.7.3 + +[Full Changelog](http://github.com/typhoeus/typhoeus/compare/v0.7.2...v0.7.3) + +* Add on_body callbacks individually to allow Ethon to recognize the return code + +## 0.7.2 + +[Full Changelog](http://github.com/typhoeus/typhoeus/compare/v0.7.1...v0.7.2) + +* Allow arrays to be passed to Expectation#and_return + ([JP Moral](https://github.com/jpmoral)) + +* Added getter for `redirect_time` value. + ([Adrien Jarthon](https://github.com/jarthod)) + +## 0.7.1 + +[Full Changelog](http://github.com/typhoeus/typhoeus/compare/v0.7.0...v0.7.1) + +Bugfixes: + +* Forking may cause libcurl sockets to be shared with child processes, causing HTTP requests to be interleaved + ([Rolf Timmermans](https://github.com/rolftimmermans), [\#436](https://github.com/typhoeus/typhoeus/pull/426)) + +## 0.7.0 + +[Full Changelog](http://github.com/typhoeus/typhoeus/compare/v0.7.0.pre1...v0.7.0) + +Bugfixes: + +* Call on_headers and on_body when using stubbed responses. + +## 0.7.0.pre1 + +[Full Changelog](http://github.com/typhoeus/typhoeus/compare/v0.6.9...v0.7.0.pre1) + +Enhancements: + +* Improving timeout behavior and documentation. `no_signal` is now set per default! + ([Jonas Wagner](https://github.com/jwagner), [\#398](https://github.com/typhoeus/typhoeus/pull/398)) + +## 0.6.8 + +[Full Changelog](http://github.com/typhoeus/typhoeus/compare/v0.6.7...v0.6.8) + +Bugfixes: + +* Fix Faraday 0.9 compatibility. + ([Gleb Mazovetskiy](https://github.com/glebm), [\#357](https://github.com/typhoeus/typhoeus/pull/357)) +* Fix Request#hash for different key orders. + ([Matthew Schulkind](https://github.com/mschulkind), [\#344](https://github.com/typhoeus/typhoeus/pull/344)) + +Enhancements: + +* Use an updated Ethon version. Note that from now on the `mime-types` is no longer a Ethon dependency. The gem will be still used if available to determine the mime type of a file which is uploaded. That means you have to have take care of the gem installation yourself. +* Use SVG for status badges in README. + ([Sean Linsley](https://github.com/seanlinsley), [\#353](https://github.com/typhoeus/typhoeus/pull/353)) +* Missing quotes in README example code. + ([Jason R. Clark](https://github.com/jasonrclark), [\#351](https://github.com/typhoeus/typhoeus/pull/351)) +* Specs for Faraday adapter. + ([michaelavila](https://github.com/michaelavila), [\#348](https://github.com/typhoeus/typhoeus/pull/348)) +* Clarify wording in README. + ([Sean Linsley](https://github.com/seanlinsley), [\#347](https://github.com/typhoeus/typhoeus/pull/347)) +* Make caching easier for non-memory caches. + ([Matthew Schulkind](https://github.com/mschulkind), [\#345](https://github.com/typhoeus/typhoeus/pull/345)) +* Spec refactoring. + ([Matthew Schulkind](https://github.com/mschulkind), [\#343](https://github.com/typhoeus/typhoeus/pull/343)) + +## 0.6.7 + +[Full Changelog](http://github.com/typhoeus/typhoeus/compare/v0.6.6...v0.6.7) + +Enhancements: + +* Add response streaming. + ([\#339](https://github.com/typhoeus/typhoeus/pull/339)) + +## 0.6.6 + +[Full Changelog](http://github.com/typhoeus/typhoeus/compare/v0.6.5...v0.6.6) + +## 0.6.5 + +[Full Changelog](http://github.com/typhoeus/typhoeus/compare/v0.6.4...v0.6.5) + +## 0.6.4 + +[Full Changelog](http://github.com/typhoeus/typhoeus/compare/v0.6.3...v0.6.4) + +The changelog entries are coming soon! + +## 0.6.3 + +[Full Changelog](http://github.com/typhoeus/typhoeus/compare/v0.6.2...v0.6.3) + +Enhancements: + +* Cache hydra per thread. +* Various documentation improvements. + ([craiglittle](https://github.com/craiglittle)) +* Add support for lazy construction of responses from stubbed requests. + ([ryankindermann](https://github.com/ryankinderman), [\#275](https://github.com/typhoeus/typhoeus/pull/275)) + +## 0.6.2 + +[Full Changelog](http://github.com/typhoeus/typhoeus/compare/v0.6.1...v0.6.2) + +Enhancements: + +* Reintroduce a global cache. +* `Request#handled_response` falls back to the original response. + ([turnerking](https://github.com/turnerking), [\#272](https://github.com/typhoeus/typhoeus/pull/272)) +* When `Errors::NoStub` is raised the `url` is displayed. + ([dschneider](https://github.com/dschneider), [\#276](https://github.com/typhoeus/typhoeus/pull/276)) +* Make `Request#hash` consistent. +* Add `.rvmrc` and `ctags` to `.gitignore`. + ([ryankindermann](https://github.com/ryankinderman), [\#274](https://github.com/typhoeus/typhoeus/pull/274)) + +## 0.6.1 + +[Full Changelog](http://github.com/typhoeus/typhoeus/compare/v0.6.0...v0.6.1) + +Enhancements: + +* Updated ethon version which allows to set multiple protocols. + +## 0.6.0 + +[Full Changelog](http://github.com/typhoeus/typhoeus/compare/v0.5.4...v0.6.0) + +Enhancements: + +* `Request#url` now also contains the url parameters. +* Use updated ethon version which provides access to protocols and redir_protocols in response to [libcurl SASL buffer overflow vulnerability](http://curl.haxx.se/docs/adv_20130206.html) + +Bugfixes: + +* Corrected ssl options for the faraday adapter. +* The before hook now correctly returns the response. + ([Mattias Putman](https://github.com/challengee), [\#268](https://github.com/typhoeus/typhoeus/pull/268)) +* Benchmark is working again. + +## 0.5.4 + +[Full Changelog](http://github.com/typhoeus/typhoeus/compare/v0.5.3...v0.5.4) + +Enhancements: + +* Make sure response_code is an integer. +* When setting an header through vcr or webmock it becomes a `Typhoeus::Response::Header`. +* Provide a Rack middleware to decode nested Typhoeus arrays properly. + ([Dwayne Macgowan](https://github.com/dwaynemac), [\#224](https://github.com/typhoeus/typhoeus/issues/224)) +* Handled response is available again. +* Rename parameter `url` to `base_url`. See discussion here: [\#250](https://github.com/typhoeus/typhoeus/issues/250). + ([bkimble](https://github.com/bkimble), [\#256](https://github.com/typhoeus/typhoeus/pull/256)) +* Provide O(1) header access. + * Work around ruby 1.8.7 limitations. + ([Chris Johnson](https://github.com/findchris), [\#227](https://github.com/typhoeus/typhoeus/pull/227) ) + * Provide symbol access. + +## 0.5.3 + +[Full Changelog](http://github.com/typhoeus/typhoeus/compare/v0.5.2...v0.5.3) + +Enhancements: + +* When checking options in Expecation#matches? also consider Request#options. + +Bugfixes: + +* Do not break backwards compatibility with case insensitive headers access. +* Make sure hydra behaves correct in case of before hooks. + +## 0.5.2 + +[Full Changelog](http://github.com/typhoeus/typhoeus/compare/v0.5.1...v0.5.2) + +Enhancements: + +* Do not check the return_code in Response#success? when response is mocked. +* Check for memoization, stubbing, before hooks are delayed to Hydra#run. It + was on Hydra#queue before and led to strange behavior because if the request + was stubbed, it was wrapped up in queue already. There was no way to add + callbacks after queue thatswhy. This is now different, since everything happens + in run, just as you expect. + +## 0.5.1 + +[Full Changelog](http://github.com/typhoeus/typhoeus/compare/v0.5.0...v0.5.1) + +Enhancements: + +* Downcase header keys for easier access + ( [\#227](https://github.com/typhoeus/typhoeus/issues/227) ) +* Using an updated Ethon version. + +## 0.5.0 + +[Full Changelog](http://github.com/typhoeus/typhoeus/compare/v0.4.2...v0.5.0) + +Major Changes: + +* Ethon integration + * Params are url params and a body is always a body for every request type + * The options you can set might have a slightly other names, as Ethon sticks to + libcurl names. See + [Easy.new](http://rubydoc.info/github/typhoeus/ethon/Ethon/Easy#initialize-instance_method) + for a description. + * Request parameter and body are properly encoded (only POST multiform body is not) + * No more header sanitizing. Before: `:headers => { 'user_agent' => 'Custom' }` was modified to + `:headers => { 'User-Agent' => 'Custom' }` + * `Typhoeus::Easy` and `Typhoeus::Multi` are now `Ethon::Easy` and `Ethon::Multi` + +* Request shortcuts: `Typhoeus.get("www.google.de")` +* Global configuration: +```ruby +Typhoeus.configure do |config| + config.verbose = true + config.memoize = true +end +``` +* No more `Response#headers_hash`, instead `Response#headers` returning the last + header and response#redirections returning the responses with headers + generated through redirections +* Instead of defining the same callbacks on every request, you can define global callbacks: +```ruby +Typhoeus.on_complete { p "yay" } +``` +* The stubbing interface changed slightly. You now have the same syntax as for requests: +```ruby +Typhoeus.stub(url, options).and_return(response) +``` +* The following things were removed because they do not seemed to be used at all. Ping me if you disagree! + * `Typhoeus::Filter` + * `Typhoeus::Remote` + * `Typhoeus::RemoteMethod` + * `Typhoeus::RemoteProxyObject` + * build in cache interface + +Enhancements: + +* Documentation + ( [Alex P](https://github.com/ifesdjeen), [\#188](https://github.com/typhoeus/typhoeus/issues/188) ) +* Request#on\_complete can hold multiple blocks. +* Request#eql? recognizes when header/params/body has a different order, but still same keys and values + ( [Alex P](https://github.com/ifesdjeen), [\#194](https://github.com/typhoeus/typhoeus/issues/194) ) + +Bug Fixes: + +* Zero bytes in strings are escaped for libcurl +* Add support for socks5 hostname proxy type + ( [eweathers](https://github.com/eweathers), [\#183](https://github.com/typhoeus/typhoeus/issues/183) ) +* Post body is encoded + ( [Rohan Deshpande](https://github.com/rdeshpande), [\#143](https://github.com/typhoeus/typhoeus/issues/143) ) +* Set default user agent + ( [Steven Shingler](https://github.com/sshingler), [\#176](https://github.com/typhoeus/typhoeus/issues/176) ) + +## 0.4.2 +* A header hotfix + +## 0.4.1 +* Fix verifypeer and verifyhost options +* Fix header sending + +## 0.4.0 +* Make a GET even when a body is given +* Deprecated User Agent setter removed +* Allow cache key basis overwrite (John Crepezzi, #147) +* FFI integration (Daniel Cavanagh, #151) +* Refactor upload code (Marnen Laibow-Koser, #152) +* Fix travis-ci build (Ezekiel Templin, #160) + +## 0.3.3 +* Make sure to call the Easy::failure callback on all non-success http response codes, even invalid ones. [balexis] +* Use bytesize instead of length to determine Content-Length [dlamacchia] +* Added SSL version option to Easy/Request [michelbarbosa/dbalatero] + +## 0.3.2 +* Fix array params to be consistent with HTTP spec [gridaphobe] +* traversal\_to\_params\_hash should use the escape option [itsmeduncan] +* Fix > 1024 open file descriptors [mschulkind] +* Fixed a bug with internally queued requests being dropped [mschulkind] +* Use gemspec in bundler to avoid duplication [mschulkind] +* Run internally queued requests in FIFO order [mschulkind] +* Moved Typhoeus::VERSION to a separate file, to fix rake build\_native [mschulkind] +* Fixed problems related to put requests with empty bodies [skaes, GH-84] +* Added CURLOPT\_INTERFACE option via Request#interface=. [spiegela] +* Added Tempfile support to Form#process! [richievos] +* Hydra won't forget to accept gzip/deflate encoding [codesnik] +* Accept and convert strings to integers in Typhoeus::Request#initialize for timeout/cache\_timeout/connect\_timeout values when using ruby 1.9.x. [djnawara] +* Added interface for registering stub finders [myronmarston] +* Fixed header stubbing [myronmarston] +* Added PKCS12 support [jodell] +* Make a request with handlers marshallable [bernerdschaefer] +* Upgraded to RSpec 2 [bernerdschaefer] +* Fix HTTP status edge-case [balexis] +* Expose primary\_ip to easy object [balexis] + +## 0.2.4 +* Fix form POSTs to only use multipart for file uploads, otherwise use application/x-www-form-urlencoded [dbalatero] + +## 0.2.3 +* Code duplication in Typhoeus::Form led to nested URL param errors on POST only. Fixed [dbalatero] + +## 0.2.2 +* Fixed a problem with nested URL params encoding incorrectly [dbalatero] + +## 0.2.1 +* Added extended proxy support [Zapotek, GH-46] +* eliminated compile time warnings by using proper type declarations [skaes, GH-54] +* fixed broken calls to rb\_raise [skaes, GH-54] +* prevent leaking of curl easy handles when exceptions are raised (either from typhoeus itself or user callbacks) [skaes, GH-54] +* fixed Easy#timed\_out? using curl return codes [skaes, GH-54] +* provide curl return codes and corresponding curl error messages on classes Easy and Request [skaes, GH-54] +* allow VCR to whitelist hosts in Typhoeus stubbing/mocking [myronmarston, GH-57] +* added timed\_out? documentation, method to Response [dbalatero, GH-34] +* added abort to Hydra to prematurely stop a hydra.run [Zapotek] +* added file upload support for POST requests [jtarchie, GH-59] + +## 0.2.0 +* Fix warning in Request#headers from attr\_accessor +* Params with array values were not parsing into the format that rack expects [GH-39, smartocci] +* Removed Rack as a dependency [GH-45] +* Added integration hooks for VCR! + +## 0.1.31 +* Fixed bug in setting compression encoding [morhekil] +* Exposed authentication control methods through Request interface [morhekil] + +## 0.1.30 +* Exposed CURLOPT\_CONNECTTIMEOUT\_MS to Requests [balexis] + +## 0.1.29 +* Fixed a memory corruption with using CURLOPT\_POSTFIELDS [gravis, +32531d0821aecc4] + +## 0.1.28 +* Added SSL cert options for Typhoeus::Easy [GH-25, gravis] +* Ported SSL cert options to Typhoeus::Request interface [gravis] +* Added support for any HTTP method (purge for Varnish) [ryana] + +## 0.1.27 +* Added rack as dependency, added dev dependencies to Rakefile [GH-21] diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..667e8a3 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,20 @@ +We love pull requests. Here's a quick guide: + +1. Fork the repo. + +2. Run the tests. We only take pull requests with passing tests, and it's great +to know that you have a clean slate: `bundle && bundle exec rake` + +3. Add a test for your change. Only refactoring and documentation changes +require no new tests. If you are adding functionality or fixing a bug, we need +a test! + +4. Make the test pass. + +5. Push to your fork and submit a pull request. + +And in case we didn't emphasize it enough: we love tests! + +## Issue triage [![Open Source Helpers](https://www.codetriage.com/typhoeus/typhoeus/badges/users.svg)](https://www.codetriage.com/typhoeus/typhoeus) + +You can contribute by triaging issues which may include reproducing bug reports or asking for vital information, such as version numbers or reproduction instructions. If you would like to start triaging issues, one easy way to get started is to [subscribe to typhoeus on CodeTriage](https://www.codetriage.com/typhoeus/typhoeus). diff --git a/Gemfile b/Gemfile new file mode 100644 index 0000000..dd662d7 --- /dev/null +++ b/Gemfile @@ -0,0 +1,32 @@ +source "https://rubygems.org" +gemspec + +if Gem.ruby_version < Gem::Version.new("2.0.0") + gem "rake", "< 11" + gem "json", "< 2" +else + gem "json" + gem "rake" +end + +group :development, :test do + gem "rspec", "~> 3.0" + + gem "sinatra", "~> 1.3" + + if Gem.ruby_version >= Gem::Version.new("1.9.0") + gem "faraday", ">= 0.9" + gem "dalli", "~> 2.0" + end + + gem "redis", "~> 3.0" + + if RUBY_PLATFORM == "java" + gem "spoon" + end + + unless ENV["CI"] + gem "guard-rspec", "~> 0.7" + gem 'rb-fsevent', '~> 0.9.1' + end +end diff --git a/Guardfile b/Guardfile new file mode 100644 index 0000000..f0db299 --- /dev/null +++ b/Guardfile @@ -0,0 +1,9 @@ +# vim:set filetype=ruby: +guard( + "rspec", + all_after_pass: false, + cli: "--fail-fast --tty --format documentation --colour") do + + watch(%r{^spec/.+_spec\.rb$}) + watch(%r{^lib/(.+)\.rb$}) { |match| "spec/#{match[1]}_spec.rb" } +end diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..387fac5 --- /dev/null +++ b/LICENSE @@ -0,0 +1,22 @@ +Copyright (c) 2009-2010 Paul Dix +Copyright (c) 2011 David Balatero +Copyright (c) 2012-2016 Hans Hasselberg + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..1bbab54 --- /dev/null +++ b/README.md @@ -0,0 +1,584 @@ +# Typhoeus [![Build Status](https://img.shields.io/travis/typhoeus/typhoeus/master.svg)](https://travis-ci.org/typhoeus/typhoeus) [![Code Climate](https://img.shields.io/codeclimate/maintainability/typhoeus/typhoeus.svg)](https://codeclimate.com/github/typhoeus/typhoeus) [![Gem Version](https://img.shields.io/gem/v/typhoeus.svg)](https://rubygems.org/gems/typhoeus) + +Like a modern code version of the mythical beast with 100 serpent heads, Typhoeus runs HTTP requests in parallel while cleanly encapsulating handling logic. + +## Example + +A single request: + +```ruby +Typhoeus.get("www.example.com", followlocation: true) +``` + +Parallel requests: + +```ruby +hydra = Typhoeus::Hydra.new +10.times.map{ hydra.queue(Typhoeus::Request.new("www.example.com", followlocation: true)) } +hydra.run +``` + +## Installation +Add the following line to your Gemfile: +``` +gem "typhoeus" +``` +Then run `bundle install` + +Or install it yourself as: + +``` +gem install typhoeus +``` + +## Project Tracking + +* [Documentation](http://rubydoc.info/github/typhoeus/typhoeus/frames/Typhoeus) (GitHub master) +* [Mailing list](http://groups.google.com/group/typhoeus) + +## Usage + +### Introduction + +The primary interface for Typhoeus is comprised of three classes: Request, Response, and Hydra. Request represents an HTTP request object, response represents an HTTP response, and Hydra manages making parallel HTTP connections. + +```ruby +request = Typhoeus::Request.new( + "www.example.com", + method: :post, + body: "this is a request body", + params: { field1: "a field" }, + headers: { Accept: "text/html" } +) +``` + +We can see from this that the first argument is the url. The second is a set of options. +The options are all optional. The default for `:method` is `:get`. + +When you want to send URL parameters, you can use `:params` hash to do so. Please note that in case of you should send a request via `x-www-form-urlencoded` parameters, you need to use `:body` hash instead. `params` are for URL parameters and `:body` is for the request body. + +#### Sending requests through the proxy + +Add a proxy url to the list of options: + +```ruby +options = {proxy: 'http://myproxy.org'} +req = Typhoeus::Request.new(url, options) +``` + +If your proxy requires authentication, add it with `proxyuserpwd` option key: + +```ruby +options = {proxy: 'http://proxyurl.com', proxyuserpwd: 'user:password'} +req = Typhoeus::Request.new(url, options) +``` + +Note that `proxyuserpwd` is a colon-separated username and password, in the vein of basic auth `userpwd` option. + + +You can run the query either on its own or through the hydra: + +``` ruby +request.run +#=> +``` + +```ruby +hydra = Typhoeus::Hydra.hydra +hydra.queue(request) +hydra.run +``` + +The response object will be set after the request is run. + +```ruby +response = request.response +response.code +response.total_time +response.headers +response.body +``` + +### Making Quick Requests + +Typhoeus has some convenience methods for performing single HTTP requests. The arguments are the same as those you pass into the request constructor. + +```ruby +Typhoeus.get("www.example.com") +Typhoeus.head("www.example.com") +Typhoeus.put("www.example.com/posts/1", body: "whoo, a body") +Typhoeus.patch("www.example.com/posts/1", body: "a new body") +Typhoeus.post("www.example.com/posts", body: { title: "test post", content: "this is my test"}) +Typhoeus.delete("www.example.com/posts/1") +Typhoeus.options("www.example.com") +``` +#### Sending params in the body with PUT +When using POST the content-type is set automatically to 'application/x-www-form-urlencoded'. That's not the case for any other method like PUT, PATCH, HEAD and so on, irrespective of whether you are using body or not. To get the same result as POST, i.e. a hash in the body coming through as params in the receiver, you need to set the content-type as shown below: +```ruby +Typhoeus.put("www.example.com/posts/1", + headers: {'Content-Type'=> "application/x-www-form-urlencoded"}, + body: {title:"test post updated title", content: "this is my updated content"} + ) +``` + +### Handling HTTP errors + +You can query the response object to figure out if you had a successful +request or not. Here’s some example code that you might use to handle errors. +The callbacks are executed right after the request is finished, make sure to define +them before running the request. + +```ruby +request = Typhoeus::Request.new("www.example.com", followlocation: true) + +request.on_complete do |response| + if response.success? + # hell yeah + elsif response.timed_out? + # aw hell no + log("got a time out") + elsif response.code == 0 + # Could not get an http response, something's wrong. + log(response.return_message) + else + # Received a non-successful http response. + log("HTTP request failed: " + response.code.to_s) + end +end + +request.run +``` + +This also works with serial (blocking) requests in the same fashion. Both +serial and parallel requests return a Response object. + +### Handling file uploads + +A File object can be passed as a param for a POST request to handle uploading +files to the server. Typhoeus will upload the file as the original file name +and use Mime::Types to set the content type. + +```ruby +Typhoeus.post( + "http://localhost:3000/posts", + body: { + title: "test post", + content: "this is my test", + file: File.open("thesis.txt","r") + } +) +``` + +### Streaming the response body + +Typhoeus can stream responses. When you're expecting a large response, +set the `on_body` callback on a request. Typhoeus will yield to the callback +with chunks of the response, as they're read. When you set an `on_body` callback, +Typhoeus will not store the complete response. + +```ruby +downloaded_file = File.open 'huge.iso', 'wb' +request = Typhoeus::Request.new("www.example.com/huge.iso") +request.on_headers do |response| + if response.code != 200 + raise "Request failed" + end +end +request.on_body do |chunk| + downloaded_file.write(chunk) +end +request.on_complete do |response| + downloaded_file.close + # Note that response.body is "" +end +request.run +``` + +If you need to interrupt the stream halfway, +you can return the `:abort` symbol from the `on_body` block, example: + +```ruby +request.on_body do |chunk| + buffer << chunk + :abort if buffer.size > 1024 * 1024 +end +``` + +This will properly stop the stream internally and avoid any memory leak which +may happen if you interrupt with something like a `return`, `throw` or `raise`. + +### Making Parallel Requests + +Generally, you should be running requests through hydra. Here is how that looks: + +```ruby +hydra = Typhoeus::Hydra.hydra + +first_request = Typhoeus::Request.new("http://example.com/posts/1") +first_request.on_complete do |response| + third_url = response.body + third_request = Typhoeus::Request.new(third_url) + hydra.queue third_request +end +second_request = Typhoeus::Request.new("http://example.com/posts/2") + +hydra.queue first_request +hydra.queue second_request +hydra.run # this is a blocking call that returns once all requests are complete +``` + +The execution of that code goes something like this. The first and second requests are built and queued. When hydra is run the first and second requests run in parallel. When the first request completes, the third request is then built and queued, in this example based on the result of the first request. The moment it is queued Hydra starts executing it. Meanwhile the second request would continue to run (or it could have completed before the first). Once the third request is done, `hydra.run` returns. + +How to get an array of response bodies back after executing a queue: + +```ruby +hydra = Typhoeus::Hydra.new +requests = 10.times.map { + request = Typhoeus::Request.new("www.example.com", followlocation: true) + hydra.queue(request) + request +} +hydra.run + +responses = requests.map { |request| + request.response.body +} +``` +`hydra.run` is a blocking request. You can also use the `on_complete` callback to handle each request as it completes: + +```ruby +hydra = Typhoeus::Hydra.new +10.times do + request = Typhoeus::Request.new("www.example.com", followlocation: true) + request.on_complete do |response| + #do_something_with response + end + hydra.queue(request) +end +hydra.run +``` + +### Making Parallel Requests with Faraday + Typhoeus + +```ruby +require 'faraday' + +conn = Faraday.new(:url => 'http://httppage.com') do |builder| + builder.request :url_encoded + builder.response :logger + builder.adapter :typhoeus +end + +conn.in_parallel do + response1 = conn.get('/first') + response2 = conn.get('/second') + + # these will return nil here since the + # requests have not been completed + response1.body + response2.body +end + +# after it has been completed the response information is fully available +# response1.status, etc +response1.body +response2.body +``` + +### Specifying Max Concurrency + +Hydra will also handle how many requests you can make in parallel. Things will get flakey if you try to make too many requests at the same time. The built in limit is 200. When more requests than that are queued up, hydra will save them for later and start the requests as others are finished. You can raise or lower the concurrency limit through the Hydra constructor. + +```ruby +Typhoeus::Hydra.new(max_concurrency: 20) +``` + +### Memoization + +Hydra memoizes requests within a single run call. You have to enable memoization. +This will result in a single request being issued. However, the on_complete handlers of both will be called. + +```ruby +Typhoeus::Config.memoize = true + +hydra = Typhoeus::Hydra.new(max_concurrency: 1) +2.times do + hydra.queue Typhoeus::Request.new("www.example.com") +end +hydra.run +``` + +This will result in two requests. + +```ruby +Typhoeus::Config.memoize = false + +hydra = Typhoeus::Hydra.new(max_concurrency: 1) +2.times do + hydra.queue Typhoeus::Request.new("www.example.com") +end +hydra.run +``` + +### Caching + +Typhoeus includes built in support for caching. In the following example, if there is a cache hit, the cached object is passed to the on_complete handler of the request object. + +```ruby +class Cache + def initialize + @memory = {} + end + + def get(request) + @memory[request] + end + + def set(request, response) + @memory[request] = response + end +end + +Typhoeus::Config.cache = Cache.new + +Typhoeus.get("www.example.com").cached? +#=> false +Typhoeus.get("www.example.com").cached? +#=> true +``` + +For use with [Dalli](https://github.com/mperham/dalli): + +```ruby +dalli = Dalli::Client.new(...) +Typhoeus::Config.cache = Typhoeus::Cache::Dalli.new(dalli) +``` + +For use with Rails: + +```ruby +Typhoeus::Config.cache = Typhoeus::Cache::Rails.new +``` + +For use with [Redis](https://github.com/redis/redis-rb): + +```ruby +redis = Redis.new(...) +Typhoeus::Config.cache = Typhoeus::Cache::Redis.new(redis) +``` + +All three of these adapters take an optional keyword argument `default_ttl`, which sets a default +TTL on cached responses (in seconds), for requests which do not have a cache TTL set. + +You may also selectively choose not to cache by setting `cache` to `false` on a request or to use +a different adapter. + +```ruby +cache = Cache.new +Typhoeus.get("www.example.com", cache: cache) +``` + +### Direct Stubbing + +Hydra allows you to stub out specific urls and patterns to avoid hitting +remote servers while testing. + +```ruby +response = Typhoeus::Response.new(code: 200, body: "{'name' : 'paul'}") +Typhoeus.stub('www.example.com').and_return(response) + +Typhoeus.get("www.example.com") == response +#=> true +``` + +The queued request will hit the stub. You can also specify a regex to match urls. + +```ruby +response = Typhoeus::Response.new(code: 200, body: "{'name' : 'paul'}") +Typhoeus.stub(/example/).and_return(response) + +Typhoeus.get("www.example.com") == response +#=> true +``` + +You may also specify an array for the stub to return sequentially. + +```ruby +Typhoeus.stub('www.example.com').and_return([response1, response2]) + +Typhoeus.get('www.example.com') == response1 #=> true +Typhoeus.get('www.example.com') == response2 #=> true +``` + +When testing make sure to clear your expectations or the stubs will persist between tests. The following can be included in your spec_helper.rb file to do this automatically. + +```ruby +RSpec.configure do |config| + config.before :each do + Typhoeus::Expectation.clear + end +end +``` + +### Timeouts + +No exceptions are raised on HTTP timeouts. You can check whether a request timed out with the following method: + +```ruby +Typhoeus.get("www.example.com", timeout: 1).timed_out? +``` + +Timed out responses also have their success? method return false. + +There are two different timeouts available: [`timeout`](http://curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTTIMEOUT) +and [`connecttimeout`](http://curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTCONNECTTIMEOUT). +`timeout` is the time limit for the entire request in seconds. +`connecttimeout` is the time limit for just the connection phase, again in seconds. + +There are two additional more fine grained options `timeout_ms` and +`connecttimeout_ms`. These options offer millisecond precision but are not always available (for instance on linux if `nosignal` is not set to true). + +When you pass a floating point `timeout` (or `connecttimeout`) Typhoeus will set `timeout_ms` for you if it has not been defined. The actual timeout values passed to curl will always be rounded up. + +DNS timeouts of less than one second are not supported unless curl is compiled with an asynchronous resolver. + +The default `timeout` is 0 (zero) which means curl never times out during transfer. The default `connecttimeout` is 300 seconds. A `connecttimeout` of 0 will also result in the default `connecttimeout` of 300 seconds. + +### Following Redirections + +Use `followlocation: true`, eg: + +```ruby +Typhoeus.get("www.example.com", followlocation: true) +``` + +### Basic Authentication + +```ruby +Typhoeus::Request.get("www.example.com", userpwd: "user:password") +``` + +### Compression + +```ruby +Typhoeus.get("www.example.com", accept_encoding: "gzip") +``` + +The above has a different behavior than setting the header directly in the header hash, eg: +```ruby +Typhoeus.get("www.example.com", headers: {"Accept-Encoding" => "gzip"}) +``` + +Setting the header hash directly will not include the `--compressed` flag in the libcurl command and therefore libcurl will not decompress the response. If you want the `--compressed` flag to be added automatically, set `:accept_encoding` Typhoeus option. + + +### Cookies + +```ruby +Typhoeus::Request.get("www.example.com", cookiefile: "/path/to/file", cookiejar: "/path/to/file") +``` + +Here, `cookiefile` is a file to read cookies from, and `cookiejar` is a file to write received cookies to. +If you just want cookies enabled, you need to pass the same filename for both options. + +### Other CURL options + +Are available and documented [here](http://rubydoc.info/github/typhoeus/ethon/Ethon/Easy/Options) + +### SSL + +SSL comes built in to libcurl so it’s in Typhoeus as well. If you pass in a +url with "https" it should just work assuming that you have your [cert +bundle](http://curl.haxx.se/docs/caextract.html) in order and the server is +verifiable. You must also have libcurl built with SSL support enabled. You can +check that by doing this: + +``` +curl --version +``` + +Now, even if you have libcurl built with OpenSSL you may still have a messed +up cert bundle or if you’re hitting a non-verifiable SSL server then you’ll +have to disable peer verification to make SSL work. Like this: + +```ruby +Typhoeus.get("https://www.example.com", ssl_verifypeer: false) +``` + +If you are getting "SSL: certificate subject name does not match target host +name" from curl (ex:- you are trying to access to b.c.host.com when the +certificate subject is \*.host.com). You can disable host verification. Like +this: + +```ruby +# host checking enabled +Typhoeus.get("https://www.example.com", ssl_verifyhost: 2) +# host checking disabled +Typhoeus.get("https://www.example.com", ssl_verifyhost: 0) +``` + +### Verbose debug output + +It’s sometimes useful to see verbose output from curl. You can enable it on a per-request basis: + +```ruby +Typhoeus.get("http://example.com", verbose: true) +``` + +or globally: + +```ruby +Typhoeus::Config.verbose = true +``` + +Just remember that libcurl prints it’s debug output to the console (to +STDERR), so you’ll need to run your scripts from the console to see it. + +### Default User Agent Header + +In many cases, all HTTP requests made by an application require the same User-Agent header set. Instead of supplying it on a per-request basis by supplying a custom header, it is possible to override it for all requests using: + + +```ruby +Typhoeus::Config.user_agent = "custom user agent" +``` + +### Running the specs + +Running the specs should be as easy as: + +``` +bundle install +bundle exec rake +``` +## Semantic Versioning + +This project conforms to [semver](http://semver.org/). + +## LICENSE + +(The MIT License) + +Copyright © 2009-2010 [Paul Dix](http://www.pauldix.net/) + +Copyright © 2011-2012 [David Balatero](https://github.com/dbalatero/) + +Copyright © 2012-2016 [Hans Hasselberg](http://github.com/i0rek/) + +Permission is hereby granted, free of charge, to any person obtaining a +copy of this software and associated documentation files (the "Software"), +to deal in the Software without restriction, including without +limitation the rights to use, copy, modify, merge, publish, distribute, +sublicense, and/or sell copies of the Software, and to permit persons +to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included +in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +OTHER DEALINGS IN THE SOFTWARE. diff --git a/Rakefile b/Rakefile new file mode 100644 index 0000000..d5693a5 --- /dev/null +++ b/Rakefile @@ -0,0 +1,38 @@ +require "bundler" +Bundler.setup + +require "rake" +require "rspec/core/rake_task" +$LOAD_PATH.unshift File.expand_path("../lib", __FILE__) +require "typhoeus/version" + +task :gem => :build +task :build do + system "gem build typhoeus.gemspec" +end + +task :install => :build do + system "gem install typhoeus-#{Typhoeus::VERSION}.gem" +end + +task :release => :build do + system "git tag -a v#{Typhoeus::VERSION} -m 'Tagging #{Typhoeus::VERSION}'" + system "git push --tags" + system "gem push typhoeus-#{Typhoeus::VERSION}.gem" +end + +RSpec::Core::RakeTask.new(:spec) do |t| + t.verbose = false + t.ruby_opts = "-W -I./spec -rspec_helper" +end + +desc "Start up the test servers" +task :start do + require_relative 'spec/support/boot' + begin + Boot.start_servers(:rake) + rescue Exception + end +end + +task :default => :spec diff --git a/UPGRADE.md b/UPGRADE.md new file mode 100644 index 0000000..28c7f1d --- /dev/null +++ b/UPGRADE.md @@ -0,0 +1,55 @@ +# Upgrade guide + +## 0.5 + +### Options + +Fix the option names, because some were renamed. The errors should point you in the right direction: + +```ruby +Typhoeus.get("www.example.com", follow_location: true) +# Ethon::Errors::InvalidOption: The option: follow_location is invalid. +# Please try followlocation instead of follow_location. +# ... [Backtrace] + +Typhoeus.get("www.example.com", followlocation: true).code +#=> 200 +``` + +### Headers + +`Response#headers` returns a hash now and replaces `Response#headers_hash`, use `Response#response_headers` for the raw string: + +```ruby +Typhoeus.get("www.example.com", followlocation: true).headers +#=> { +# "date"=>"Tue, 06 Nov 2012 09:07:27 GMT", +# "server"=>"Apache/2.2.3 (CentOS)", +# "last-modified"=>"Wed, 09 Feb 2011 17:13:15 GMT", +# "vary"=>"Accept-Encoding", +# "connection"=>"close", +# "content-type"=>"text/html; charset=UTF-8" +# } + +Typhoeus.get("www.example.com", followlocation: true).response_headers +#=> "HTTP/1.0 302 Found\r\nLocation: http://www.iana.org/domains/example/ [...]" +``` + +### Params vs body + +Make sure every request sends proper params and body (especially POST/PUT). `:params` becomes url parameter and `:body` request body. Before params for POST was smashed into the body. + +### Configuration + +Create a global configuration in case you want to turn on verbose, memoize or block_connection: + +```ruby +Typhoeus.configure do |config| + config.verbose = true + config.memoize = true +end +``` + +### Docs + +When in doubt, read the [docs](http://rubydoc.info/github/typhoeus/typhoeus/frames/Typhoeus) or the [code](https://www.github.com/typhoeus). diff --git a/lib/rack/typhoeus.rb b/lib/rack/typhoeus.rb new file mode 100644 index 0000000..dd93611 --- /dev/null +++ b/lib/rack/typhoeus.rb @@ -0,0 +1 @@ +require "rack/typhoeus/middleware/params_decoder" diff --git a/lib/rack/typhoeus/middleware/params_decoder.rb b/lib/rack/typhoeus/middleware/params_decoder.rb new file mode 100644 index 0000000..cbd10f3 --- /dev/null +++ b/lib/rack/typhoeus/middleware/params_decoder.rb @@ -0,0 +1,57 @@ +require 'rack/typhoeus/middleware/params_decoder/helper' + +module Rack + module Typhoeus + module Middleware + + # This Rack middleware takes care of the proper deserialization of + # the nested params encoded by Typhoeus. + # + # @example Require the railtie when using Rails. + # require 'typhoeus/railtie' + # + # @example Include the middleware for Rack based applications. + # use Rack::Typhoeus::Middleware::ParamsDecoder + # + # @example Use the helper directly. Not recommended as b/c the interface might change. + # require 'rack/typhoeus/middleware/params_decoder/helper' + # include Rack::Typhoeus::Middleware::ParamsDecoder::Helper + # decode!(params) + # + # @author Dwayne Macgowan + # @since 0.5.4 + class ParamsDecoder + include ParamsDecoder::Helper + + def initialize(app) + @app = app + end + + def call(env) + req = Rack::Request.new(env) + decode(req.params).each_pair { |k, v| update_params req, k, v } + @app.call(env) + end + + private + + # Persist params change in environment. Extracted from: + # https://github.com/rack/rack/blob/master/lib/rack/request.rb#L243 + def update_params(req, k, v) + found = false + if req.GET.has_key?(k) + found = true + req.GET[k] = v + end + if req.POST.has_key?(k) + found = true + req.POST[k] = v + end + unless found + req.GET[k] = v + end + end + end + end + end +end diff --git a/lib/rack/typhoeus/middleware/params_decoder/helper.rb b/lib/rack/typhoeus/middleware/params_decoder/helper.rb new file mode 100644 index 0000000..0542328 --- /dev/null +++ b/lib/rack/typhoeus/middleware/params_decoder/helper.rb @@ -0,0 +1,76 @@ +module Rack + module Typhoeus + module Middleware + class ParamsDecoder + module Helper + + # Recursively decodes Typhoeus encoded arrays in given Hash. + # + # @example Use directly in a Rails controller. + # class ApplicationController + # before_filter :decode_typhoeus_arrays + # end + # + # @author Dwayne Macgowan + # + def decode_typhoeus_arrays + decode!(params) + end + + # Recursively decodes Typhoeus encoded arrays in given Hash. + # + # @param hash [Hash]. This Hash will be modified! + # + # @return [Hash] Hash with properly decoded nested arrays. + def decode!(hash) + return hash unless hash.is_a?(Hash) + hash.each_pair do |key,value| + if value.is_a?(Hash) + decode!(value) + hash[key] = convert(value) + end + end + hash + end + + def decode(hash) + decode!(hash.dup) + end + + private + + # Checks if Hash is an Array encoded as a Hash. + # Specifically will check for the Hash to have this + # form: {'0' => v0, '1' => v1, .., 'n' => vN } + # + # @param hash [Hash] + # + # @return [Boolean] True if its a encoded Array, else false. + def encoded?(hash) + return false if hash.empty? + if hash.keys.size > 1 + keys = hash.keys.map{|i| i.to_i if i.respond_to?(:to_i)}.sort + keys == hash.keys.size.times.to_a + else + hash.keys.first =~ /0/ + end + end + + # If the Hash is an array encoded by typhoeus an array is returned + # else the self is returned + # + # @param hash [Hash] The Hash to convert into an Array. + # + # @return [Arraya/Hash] + def convert(hash) + if encoded?(hash) + hash.sort{ |a, b| a[0].to_i <=> b[0].to_i }.map{ |key, value| value } + else + hash + end + end + end + end + end + end +end diff --git a/lib/typhoeus.rb b/lib/typhoeus.rb new file mode 100644 index 0000000..1e2226a --- /dev/null +++ b/lib/typhoeus.rb @@ -0,0 +1,143 @@ +require 'digest/sha2' +require 'ethon' + +require 'typhoeus/config' +require 'typhoeus/easy_factory' +require 'typhoeus/errors' +require 'typhoeus/expectation' +require 'typhoeus/hydra' +require 'typhoeus/pool' +require 'typhoeus/request' +require 'typhoeus/response' +require 'typhoeus/version' + +# If we are using any Rack-based application, then we need the Typhoeus rack +# middleware to ensure our app is running properly. +if defined?(Rack) + require "rack/typhoeus" +end + +# If the Redis gem is available, load the redis cache adapter +if defined?(Redis) + require "typhoeus/cache/redis" +end + +# If the Dalli gem is available, load the Dalli cache adapter +if defined?(Dalli) + require "typhoeus/cache/dalli" +end + +# If we are using Rails, load the Rails cache adapter +if defined?(Rails) + require "typhoeus/cache/rails" +end + +# If we are using Rails, then we will include the Typhoeus railtie. +# if defined?(Rails) +# require "typhoeus/railtie" +# end + +# Typhoeus is a HTTP client library based on Ethon which +# wraps libcurl. Sitting on top of libcurl makes Typhoeus +# very reliable and fast. +# +# There are some gems using Typhoeus like +# {https://github.com/myronmarston/vcr VCR}, +# {https://github.com/bblimke/webmock WebMock} or +# {https://github.com/technoweenie/faraday Faraday}. VCR +# and WebMock provide their own adapter whereas +# Faraday relies on {Faraday::Adapter::Typhoeus} +# since Typhoeus version 0.5. +# +# @example (see Typhoeus::Request) +# @example (see Typhoeus::Hydra) +# +# @see Typhoeus::Request +# @see Typhoeus::Hydra +# @see Faraday::Adapter::Typhoeus +# +# @since 0.5.0 +module Typhoeus + extend Request::Actions + extend Request::Callbacks::Types + + # The default Typhoeus user agent. + USER_AGENT = "Typhoeus - https://github.com/typhoeus/typhoeus" + + # Set the Typhoeus configuration options by passing a block. + # + # @example (see Typhoeus::Config) + # + # @yield [ Typhoeus::Config ] + # + # @return [ Typhoeus::Config ] The configuration. + # + # @see Typhoeus::Config + def self.configure + yield Config + end + + # Stub out a specific request. + # + # @example (see Typhoeus::Expectation) + # + # @param [ String ] base_url The url to stub out. + # @param [ Hash ] options The options to stub out. + # + # @return [ Typhoeus::Expectation ] The expecatation. + # + # @see Typhoeus::Expectation + def self.stub(base_url, options = {}, &block) + expectation = Expectation.all.find{ |e| e.base_url == base_url && e.options == options } + if expectation.nil? + expectation = Expectation.new(base_url, options) + Expectation.all << expectation + end + + expectation.and_return(&block) unless block.nil? + expectation + end + + # Add before callbacks. + # + # @example Add before callback. + # Typhoeus.before { |request| p request.base_url } + # + # @param [ Block ] block The callback. + # + # @yield [ Typhoeus::Request ] + # + # @return [ Array ] All before blocks. + def self.before(&block) + @before ||= [] + @before << block if block_given? + @before + end + + # Execute given block as if block connection is turned off. + # The old block connection state is restored afterwards. + # + # @example Make a real request, no matter if it's blocked. + # Typhoeus::Config.block_connection = true + # Typhoeus.get("www.example.com").code + # #=> raise Typhoeus::Errors::NoStub + # + # Typhoeus.with_connection do + # Typhoeus.get("www.example.com").code + # #=> :ok + # end + # + # @yield Yields control to the block after disabling block_connection. + # Afterwards, the block_connection is set to its original + # value. + # @return [ Object ] Returns the return value of the block. + # + # @see Typhoeus::Config.block_connection + def self.with_connection + old = Config.block_connection + Config.block_connection = false + result = yield if block_given? + Config.block_connection = old + result + end +end diff --git a/lib/typhoeus/adapters/faraday.rb b/lib/typhoeus/adapters/faraday.rb new file mode 100644 index 0000000..8452010 --- /dev/null +++ b/lib/typhoeus/adapters/faraday.rb @@ -0,0 +1,180 @@ +require 'faraday' + +module Faraday # :nodoc: + class Adapter # :nodoc: + + # Adapter to use Faraday with Typhoeus. + # + # @example Use Typhoeus. + # require 'faraday' + # require 'typhoeus' + # require 'typhoeus/adapters/faraday' + # + # conn = Faraday.new(url: "www.example.com") do |faraday| + # faraday.adapter :typhoeus + # + # # You can include Typhoeus options to be used for every request + # # faraday.adapter :typhoeus, forbid_reuse: true, maxredirs: 1 + # end + # + # response = conn.get("/") + class Typhoeus < Faraday::Adapter + self.supports_parallel = true + + (class << self; self; end).instance_eval do + remove_method :setup_parallel_manager if method_defined? :setup_parallel_manager + end + + remove_method :call if method_defined? :call + remove_method :perform_request if method_defined? :perform_request + remove_method :request if method_defined? :request + remove_method :read_body if method_defined? :read_body + remove_method :configure_ssl if method_defined? :configure_ssl + remove_method :configure_proxy if method_defined? :configure_proxy + remove_method :configure_timeout if method_defined? :configure_timeout + remove_method :configure_socket if method_defined? :configure_socket + remove_method :parallel? if method_defined? :parallel? + + # Initialize the Typhoeus adapter + # + # @param [ App ] app Farday app + # @option [ Hash ] adapter_options Typhoeus options + # + # @return [ void ] + def initialize(app, adapter_options = {}) + super(app) + @adapter_options = adapter_options + end + + # Setup Hydra with provided options. + # + # @example Setup Hydra. + # Faraday::Adapter::Typhoeus.setup_parallel_manager + # #=> # + # + # @param (see Typhoeus::Hydra#initialize) + # @option (see Typhoeus::Hydra#initialize) + # + # @return [ Typhoeus::Hydra ] The hydra. + def self.setup_parallel_manager(options = {}) + ::Typhoeus::Hydra.new(options) + end + + dependency 'typhoeus' + + # Hook into Faraday and perform the request with Typhoeus. + # + # @param [ Hash ] env The environment. + # + # @return [ void ] + def call(env) + super + perform_request env + @app.call env + end + + private + + def perform_request(env) + if parallel?(env) + env[:parallel_manager].queue request(env) + else + request(env).run + end + end + + def request(env) + read_body env + + req = typhoeus_request(env) + + configure_ssl req, env + configure_proxy req, env + configure_timeout req, env + configure_socket req, env + + req.on_complete do |resp| + if resp.timed_out? + env[:typhoeus_timed_out] = true + unless parallel?(env) + raise Faraday::TimeoutError, "request timed out" + end + elsif (resp.response_code == 0) || ((resp.return_code != :ok) && !resp.mock?) + env[:typhoeus_connection_failed] = true + env[:typhoeus_return_message] = resp.return_message + unless parallel?(env) + raise Faraday::ConnectionFailed, resp.return_message + end + end + + save_response(env, resp.code, resp.body) do |response_headers| + response_headers.parse resp.response_headers + end + # in async mode, :response is initialized at this point + env[:response].finish(env) if parallel?(env) + end + + req + end + + def typhoeus_request(env) + opts = { + :method => env[:method], + :body => env[:body], + :headers => env[:request_headers] + }.merge(@adapter_options) + + ::Typhoeus::Request.new(env[:url].to_s, opts) + end + + def read_body(env) + env[:body] = env[:body].read if env[:body].respond_to? :read + end + + def configure_ssl(req, env) + ssl = env[:ssl] + + verify_p = (ssl && ssl.fetch(:verify, true)) + + ssl_verifyhost = verify_p ? 2 : 0 + req.options[:ssl_verifyhost] = ssl_verifyhost + req.options[:ssl_verifypeer] = verify_p + req.options[:sslversion] = ssl[:version] if ssl[:version] + req.options[:sslcert] = ssl[:client_cert] if ssl[:client_cert] + req.options[:sslkey] = ssl[:client_key] if ssl[:client_key] + req.options[:cainfo] = ssl[:ca_file] if ssl[:ca_file] + req.options[:capath] = ssl[:ca_path] if ssl[:ca_path] + client_cert_passwd_key = [:client_cert_passwd, :client_certificate_password].detect { |name| ssl.key?(name) } + req.options[:keypasswd] = ssl[client_cert_passwd_key] if client_cert_passwd_key + end + + def configure_proxy(req, env) + proxy = env[:request][:proxy] + return unless proxy + + req.options[:proxy] = "#{proxy[:uri].scheme}://#{proxy[:uri].host}:#{proxy[:uri].port}" + + if proxy[:user] && proxy[:password] + req.options[:proxyauth] = :any + req.options[:proxyuserpwd] = "#{proxy[:user]}:#{proxy[:password]}" + end + end + + def configure_timeout(req, env) + env_req = env[:request] + req.options[:timeout_ms] = (env_req[:timeout] * 1000).to_i if env_req[:timeout] + req.options[:connecttimeout_ms] = (env_req[:open_timeout] * 1000).to_i if env_req[:open_timeout] + end + + def configure_socket(req, env) + if bind = env[:request][:bind] + req.options[:interface] = bind[:host] + end + end + + def parallel?(env) + !!env[:parallel_manager] + end + end + end +end diff --git a/lib/typhoeus/cache/dalli.rb b/lib/typhoeus/cache/dalli.rb new file mode 100644 index 0000000..5a09b10 --- /dev/null +++ b/lib/typhoeus/cache/dalli.rb @@ -0,0 +1,28 @@ +module Typhoeus + module Cache + # This module provides a simple way to cache HTTP responses using Dalli. + class Dalli + # @example Set Dalli as the Typhoeus cache backend + # Typhoeus::Config.cache = Typhoeus::Cache::Dalli.new + # + # @param [ Dalli::Client ] client + # A connection to the cache server. Defaults to `Dalli::Client.new` + # @param [ Hash ] options + # Options + # @option options [ Integer ] :default_ttl + # The default TTL of cached responses in seconds, for requests which do not set a cache_ttl. + def initialize(client = ::Dalli::Client.new, options = {}) + @client = client + @default_ttl = options[:default_ttl] + end + + def get(request) + @client.get(request.cache_key) + end + + def set(request, response) + @client.set(request.cache_key, response, request.cache_ttl || @default_ttl) + end + end + end +end diff --git a/lib/typhoeus/cache/rails.rb b/lib/typhoeus/cache/rails.rb new file mode 100644 index 0000000..10835e7 --- /dev/null +++ b/lib/typhoeus/cache/rails.rb @@ -0,0 +1,28 @@ +module Typhoeus + module Cache + # This module provides a simple way to cache HTTP responses in using the Rails cache. + class Rails + # @example Use the Rails cache setup to cache Typhoeus responses. + # Typhoeus::Config.cache = Typhoeus::Cache::Rails.new + # + # @param [ ActiveSupport::Cache::Store ] cache + # A Rails cache backend. Defaults to Rails.cache. + # @param [ Hash ] options + # Options + # @option options [ Integer ] :default_ttl + # The default TTL of cached responses in seconds, for requests which do not set a cache_ttl. + def initialize(cache = ::Rails.cache, options = {}) + @cache = cache + @default_ttl = options[:default_ttl] + end + + def get(request) + @cache.read(request) + end + + def set(request, response) + @cache.write(request.cache_key, response, :expires_in => request.cache_ttl || @default_ttl) + end + end + end +end diff --git a/lib/typhoeus/cache/redis.rb b/lib/typhoeus/cache/redis.rb new file mode 100644 index 0000000..e54ca4e --- /dev/null +++ b/lib/typhoeus/cache/redis.rb @@ -0,0 +1,35 @@ +module Typhoeus + module Cache + # This module provides a simple way to cache HTTP responses in Redis. + class Redis + # @example Set Redis as the Typhoeus cache backend + # Typhoeus::Config.cache = Typhoeus::Cache::Redis.new + # + # @param [ Redis ] redis + # A connection to Redis. Defaults to `Redis.new`, which uses the + # `REDIS_URL` environment variable to connect + # @param [ Hash ] options + # Options + # @option options [ Integer ] :default_ttl + # The default TTL of cached responses in seconds, for requests which do not set a cache_ttl. + def initialize(redis = ::Redis.new, options = {}) + @redis = redis + @default_ttl = options[:default_ttl] + end + + def get(request) + serialized_response = @redis.get(request.cache_key) + return unless serialized_response + Marshal.load(serialized_response) + end + + def set(request, response) + ttl = request.cache_ttl || @default_ttl + key = request.cache_key + serialized_response = Marshal.dump(response) + @redis.set(key, serialized_response) + @redis.expire(key, ttl) if ttl + end + end + end +end diff --git a/lib/typhoeus/config.rb b/lib/typhoeus/config.rb new file mode 100644 index 0000000..f05491c --- /dev/null +++ b/lib/typhoeus/config.rb @@ -0,0 +1,69 @@ +module Typhoeus + + # The Typhoeus configuration used to set global + # options. + # @example Set the configuration options within a block. + # Typhoeus.configure do |config| + # config.verbose = true + # end + # + # @example Set the configuration directly. + # Typhoeus::Config.verbose = true + module Config + extend self + + # Defines whether the connection is blocked. + # Defaults to false. When set to true, only + # stubbed requests are allowed. A + # {Typhoeus::Errors::NoStub} error is raised, + # when trying to do a real request. It's possible + # to work around inside + # {Typhoeus.with_connection}. + # + # @return [ Boolean ] + # + # @see Typhoeus::Request::BlockConnection + # @see Typhoeus::Hydra::BlockConnection + # @see Typhoeus#with_connection + # @see Typhoeus::Errors::NoStub + attr_accessor :block_connection + + # Defines whether GET requests are memoized when using the {Typhoeus::Hydra}. + # + # @return [ Boolean ] + # + # @see Typhoeus::Hydra + # @see Typhoeus::Hydra::Memoizable + attr_accessor :memoize + + # Defines whether curls debug output is shown. + # Unfortunately it prints to stderr. + # + # @return [ Boolean ] + # + # @see http://curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTVERBOSE + attr_accessor :verbose + + # Defines whether requests are cached. + # + # @return [ Object ] + # + # @see Typhoeus::Hydra::Cacheable + # @see Typhoeus::Request::Cacheable + attr_accessor :cache + + # Defines whether to use a default user agent. + # + # @return [ String ] + # + # @see Typhoeus::Request#set_defaults + attr_accessor :user_agent + + # Defines wether to use a proxy server for every request. + # + # @return [ String ] + # + # @see Typhoeus::Request#set_defaults + attr_accessor :proxy + end +end diff --git a/lib/typhoeus/easy_factory.rb b/lib/typhoeus/easy_factory.rb new file mode 100644 index 0000000..84c7131 --- /dev/null +++ b/lib/typhoeus/easy_factory.rb @@ -0,0 +1,180 @@ +require 'set' + +module Typhoeus + + # This is a Factory for easies to be used in the hydra. + # Before an easy is ready to be added to a multi the + # on_complete callback to be set. + # This is done by this class. + # + # @api private + class EasyFactory + + RENAMED_OPTIONS = { + :auth_method => :httpauth, + :connect_timeout => :connecttimeout, + :encoding => :accept_encoding, + :follow_location => :followlocation, + :max_redirects => :maxredirs, + :proxy_type => :proxytype, + :ssl_cacert => :cainfo, + :ssl_capath => :capath, + :ssl_cert => :sslcert, + :ssl_cert_type => :sslcerttype, + :ssl_key => :sslkey, + :ssl_key_password => :keypasswd, + :ssl_key_type => :sslkeytype, + :ssl_version => :sslversion, + } + + CHANGED_OPTIONS = { + :disable_ssl_host_verification => :ssl_verifyhost, + :disable_ssl_peer_verification => :ssl_verifypeer, + :proxy_auth_method => :proxyauth, + } + + REMOVED_OPTIONS = Set.new([:cache_key_basis, :cache_timeout, :user_agent]) + + SANITIZE_IGNORE = Set.new([:method, :cache_ttl, :cache]) + SANITIZE_TIMEOUT = Set.new([:timeout_ms, :connecttimeout_ms]) + + # Returns the request provided. + # + # @return [ Typhoeus::Request ] + attr_reader :request + + # Returns the hydra provided. + # + # @return [ Typhoeus::Hydra ] + attr_reader :hydra + + # Create an easy factory. + # + # @example Create easy factory. + # Typhoeus::Hydra::EasyFactory.new(request, hydra) + # + # @param [ Request ] request The request to build an easy for. + # @param [ Hydra ] hydra The hydra to build an easy for. + def initialize(request, hydra = nil) + @request = request + @hydra = hydra + end + + # Return the easy in question. + # + # @example Return easy. + # easy_factory.easy + # + # @return [ Ethon::Easy ] The easy. + def easy + @easy ||= Typhoeus::Pool.get + end + + # Fabricated easy. + # + # @example Prepared easy. + # easy_factory.get + # + # @return [ Ethon::Easy ] The easy. + def get + begin + easy.http_request( + request.base_url.to_s, + request.options.fetch(:method, :get), + sanitize(request.options) + ) + rescue Ethon::Errors::InvalidOption => e + help = provide_help(e.message.match(/:\s(\w+)/)[1]) + raise $!, "#{$!}#{help}", $!.backtrace + end + set_callback + easy + end + + private + + def sanitize(options) + # set nosignal to true by default + # this improves thread safety and timeout behavior + sanitized = {:nosignal => true} + options.each do |k,v| + s = k.to_sym + next if SANITIZE_IGNORE.include?(s) + if new_option = RENAMED_OPTIONS[k.to_sym] + warn("Deprecated option #{k}. Please use #{new_option} instead.") + sanitized[new_option] = v + # sanitize timeouts + elsif SANITIZE_TIMEOUT.include?(s) + if !v.integer? + warn("Value '#{v}' for option '#{k}' must be integer.") + end + sanitized[k] = v.ceil + else + sanitized[k] = v + end + end + + sanitize_timeout!(sanitized, :timeout) + sanitize_timeout!(sanitized, :connecttimeout) + + sanitized + end + + def sanitize_timeout!(options, timeout) + timeout_ms = :"#{timeout}_ms" + if options[timeout] && options[timeout].round != options[timeout] + if !options[timeout_ms] + options[timeout_ms] = (options[timeout]*1000).ceil + end + options[timeout] = options[timeout].ceil + end + options + end + + # Sets on_complete callback on easy in order to be able to + # track progress. + # + # @example Set callback. + # easy_factory.set_callback + # + # @return [ Ethon::Easy ] The easy. + def set_callback + if request.streaming? + response = nil + easy.on_headers do |easy| + response = Response.new(Ethon::Easy::Mirror.from_easy(easy).options) + request.execute_headers_callbacks(response) + end + request.on_body.each do |callback| + easy.on_body do |chunk, easy| + callback.call(chunk, response) + end + end + else + easy.on_headers do |easy| + request.execute_headers_callbacks(Response.new(Ethon::Easy::Mirror.from_easy(easy).options)) + end + end + request.on_progress.each do |callback| + easy.on_progress do |dltotal, dlnow, ultotal, ulnow, easy| + callback.call(dltotal, dlnow, ultotal, ulnow, response) + end + end + easy.on_complete do |easy| + request.finish(Response.new(easy.mirror.options)) + Typhoeus::Pool.release(easy) + if hydra && !hydra.queued_requests.empty? + hydra.dequeue_many + end + end + end + + def provide_help(option) + if new_option = CHANGED_OPTIONS[option.to_sym] + "\nPlease try #{new_option} instead of #{option}." if new_option + elsif REMOVED_OPTIONS.include?(option.to_sym) + "\nThe option #{option} was removed." + end + end + end +end diff --git a/lib/typhoeus/errors.rb b/lib/typhoeus/errors.rb new file mode 100644 index 0000000..5ad2c5d --- /dev/null +++ b/lib/typhoeus/errors.rb @@ -0,0 +1,9 @@ +require 'typhoeus/errors/typhoeus_error' +require 'typhoeus/errors/no_stub' + +module Typhoeus + + # This namespace contains all errors raised by Typhoeus. + module Errors + end +end diff --git a/lib/typhoeus/errors/no_stub.rb b/lib/typhoeus/errors/no_stub.rb new file mode 100644 index 0000000..d04534c --- /dev/null +++ b/lib/typhoeus/errors/no_stub.rb @@ -0,0 +1,12 @@ +module Typhoeus + module Errors + + # Raises when block connection is turned on + # and making a real request. + class NoStub < TyphoeusError + def initialize(request) + super("The connection is blocked and no stub defined: #{request.url}") + end + end + end +end diff --git a/lib/typhoeus/errors/typhoeus_error.rb b/lib/typhoeus/errors/typhoeus_error.rb new file mode 100644 index 0000000..1550b40 --- /dev/null +++ b/lib/typhoeus/errors/typhoeus_error.rb @@ -0,0 +1,8 @@ +module Typhoeus + module Errors + + # Default typhoeus error class for all custom errors. + class TyphoeusError < StandardError + end + end +end diff --git a/lib/typhoeus/expectation.rb b/lib/typhoeus/expectation.rb new file mode 100644 index 0000000..a8ae368 --- /dev/null +++ b/lib/typhoeus/expectation.rb @@ -0,0 +1,217 @@ +module Typhoeus + + # This class represents an expectation. It is part + # of the stubbing mechanism. An expectation contains + # a url and options, like a request. They are compared + # to the request url and options in order to evaluate + # whether they match. If that's the case, the attached + # responses are returned one by one. + # + # @example Stub a request and get specified response. + # expected = Typhoeus::Response.new + # Typhoeus.stub("www.example.com").and_return(expected) + # + # actual = Typhoeus.get("www.example.com") + # expected == actual + # #=> true + # + # @example Stub a request and get a lazily-constructed response containing data from actual widgets that exist in the system when the stubbed request is made. + # Typhoeus.stub("www.example.com/widgets") do + # actual_widgets = Widget.all + # Typhoeus::Response.new( + # :body => actual_widgets.inject([]) do |ids, widget| + # ids << widget.id + # end.join(",") + # ) + # end + # + # @example Stub a request and get a lazily-constructed response in the format requested. + # Typhoeus.stub("www.example.com") do |request| + # accept = (request.options[:headers]||{})['Accept'] || "application/json" + # format = accept.split(",").first + # body_obj = { 'things' => [ { 'id' => 'foo' } ] } + # + # Typhoeus::Response.new( + # :headers => { + # 'Content-Type' => format + # }, + # :body => SERIALIZERS[format].serialize(body_obj) + # ) + # end + class Expectation + + # @api private + attr_reader :base_url + + # @api private + attr_reader :options + + # @api private + attr_reader :from + + class << self + + # Returns all expectations. + # + # @example Return expectations. + # Typhoeus::Expectation.all + # + # @return [ Array ] The expectations. + def all + @expectations ||= [] + end + + # Clears expectations. This is handy while + # testing, and you want to make sure that + # you don't get canned responses. + # + # @example Clear expectations. + # Typhoeus::Expectation.clear + def clear + all.clear + end + + # Returns stubbed response matching the + # provided request. + # + # @example Find response + # Typhoeus::Expectation.response_for(request) + # + # @return [ Typhoeus::Response ] The stubbed response from a + # matching expectation, or nil if no matching expectation + # is found. + # + # @api private + def response_for(request) + expectation = find_by(request) + return nil if expectation.nil? + + expectation.response(request) + end + + # @api private + def find_by(request) + all.find do |expectation| + expectation.matches?(request) + end + end + end + + # Creates an expectation. + # + # @example Create expectation. + # Typhoeus::Expectation.new(base_url) + # + # @return [ Expectation ] The created expectation. + # + # @api private + def initialize(base_url, options = {}) + @base_url = base_url + @options = options + @response_counter = 0 + @from = nil + end + + # Set from value to mark an expectaion. Useful for + # other libraries, e.g. WebMock. + # + # @example Mark expectation. + # expectation.from(:webmock) + # + # @param [ String ] value Value to set. + # + # @return [ Expectation ] Returns self. + # + # @api private + def stubbed_from(value) + @from = value + self + end + + # Specify what should be returned, + # when this expectation is hit. + # + # @example Add response. + # expectation.and_return(response) + # + # @return [ void ] + def and_return(response=nil, &block) + new_response = (response.nil? ? block : response) + responses.push(*new_response) + end + + # Checks whether this expectation matches + # the provided request. + # + # @example Check if request matches. + # expectation.matches? request + # + # @param [ Request ] request The request to check. + # + # @return [ Boolean ] True when matches, else false. + # + # @api private + def matches?(request) + url_match?(request.base_url) && options_match?(request) + end + + # Return canned responses. + # + # @example Return responses. + # expectation.responses + # + # @return [ Array ] The responses. + # + # @api private + def responses + @responses ||= [] + end + + # Return the response. When there are + # multiple responses, they are returned one + # by one. + # + # @example Return response. + # expectation.response + # + # @return [ Response ] The response. + # + # @api private + def response(request) + response = responses.fetch(@response_counter, responses.last) + if response.respond_to?(:call) + response = response.call(request) + end + @response_counter += 1 + response.mock = @from || true + response + end + + private + + # Check whether the options matches the request options. + # I checks options and original options. + def options_match?(request) + (options ? options.all?{ |k,v| request.original_options[k] == v || request.options[k] == v } : true) + end + + # Check whether the base_url matches the request url. + # The base_url can be a string, regex or nil. String and + # regexp are checked, nil is always true, else false. + # + # Nil serves as a placeholder in case you want to match + # all urls. + def url_match?(request_url) + case base_url + when String + base_url == request_url + when Regexp + base_url === request_url + when nil + true + else + false + end + end + end +end diff --git a/lib/typhoeus/hydra.rb b/lib/typhoeus/hydra.rb new file mode 100644 index 0000000..0796981 --- /dev/null +++ b/lib/typhoeus/hydra.rb @@ -0,0 +1,95 @@ +require 'typhoeus/hydra/addable' +require 'typhoeus/hydra/before' +require 'typhoeus/hydra/cacheable' +require 'typhoeus/hydra/block_connection' +require 'typhoeus/hydra/memoizable' +require 'typhoeus/hydra/queueable' +require 'typhoeus/hydra/runnable' +require 'typhoeus/hydra/stubbable' + +module Typhoeus + + # Hydra manages making parallel HTTP requests. This + # is achieved by using libcurls multi interface: + # http://curl.haxx.se/libcurl/c/libcurl-multi.html + # The benefits are that you don't have to worry running + # the requests by yourself. + # + # Hydra will also handle how many requests you can + # make in parallel. Things will get flakey if you + # try to make too many requests at the same time. + # The built in limit is 200. When more requests than + # that are queued up, hydra will save them for later + # and start the requests as others are finished. You + # can raise or lower the concurrency limit through + # the Hydra constructor. + # + # Regarding the asynchronous behavior of the hydra, + # it is important to know that this is completely hidden + # from the developer and you are free to apply + # whatever technique you want to your code. That should not + # conflict with libcurls internal concurrency mechanism. + # + # @example Use the hydra to do multiple requests. + # hydra = Typhoeus::Hydra.new + # requests = (0..9).map{ Typhoeus::Request.new("www.example.com") } + # requests.each{ |request| hydra.queue(request) } + # hydra.run + # + # @note Callbacks are going to delay the request + # execution. + class Hydra + include Hydra::Addable + include Hydra::Runnable + include Hydra::Memoizable + include Hydra::Cacheable + include Hydra::BlockConnection + include Hydra::Stubbable + include Hydra::Before + include Hydra::Queueable + + # @example Set max_concurrency. + # Typhoeus::Hydra.new(max_concurrency: 20) + attr_accessor :max_concurrency + + # @api private + attr_reader :multi + + class << self + + # Returns a memoized hydra instance. + # + # @example Get a hydra. + # Typhoeus::Hydra.hydra + # + # @return [Typhoeus::Hydra] A new hydra. + def hydra + Thread.current[:typhoeus_hydra] ||= new + end + end + + # Create a new hydra. All + # {http://rubydoc.info/github/typhoeus/ethon/Ethon/Multi#initialize-instance_method Ethon::Multi#initialize} + # options are also available. + # + # @example Create a hydra. + # Typhoeus::Hydra.new + # + # @example Create a hydra with max_concurrency. + # Typhoeus::Hydra.new(max_concurrency: 20) + # + # @param [ Hash ] options The options hash. + # + # @option options :max_concurrency [ Integer ] Number + # of max concurrent connections to create. Default is + # 200. + # + # @see http://rubydoc.info/github/typhoeus/ethon/Ethon/Multi#initialize-instance_method + # Ethon::Multi#initialize + def initialize(options = {}) + @options = options + @max_concurrency = Integer(@options.fetch(:max_concurrency, 200)) + @multi = Ethon::Multi.new(options.reject{|k,_| k==:max_concurrency}) + end + end +end diff --git a/lib/typhoeus/hydra/addable.rb b/lib/typhoeus/hydra/addable.rb new file mode 100644 index 0000000..1a86960 --- /dev/null +++ b/lib/typhoeus/hydra/addable.rb @@ -0,0 +1,23 @@ +module Typhoeus + class Hydra + + # This module handles the request adding on + # hydra. + # + # @api private + module Addable + + # Adds request to multi. + # + # @example Add request. + # hydra.add(request) + # + # @param [ Typhoeus::Request ] request to add. + # + # @return [ void ] + def add(request) + multi.add(EasyFactory.new(request, self).get) + end + end + end +end diff --git a/lib/typhoeus/hydra/before.rb b/lib/typhoeus/hydra/before.rb new file mode 100644 index 0000000..75e9213 --- /dev/null +++ b/lib/typhoeus/hydra/before.rb @@ -0,0 +1,31 @@ +module Typhoeus + class Hydra + + # This module provides a way to hook into before + # a request gets queued in hydra. This is very powerful + # and you should be careful because when you accidently + # return a falsy value the request won't be executed. + # + # @api private + module Before + + # Overrride add in order to execute callbacks in + # Typhoeus.before. Will break and return when a + # callback returns nil, false or a response. Calls super + # otherwise. + # + # @example Add the request. + # hydra.add(request) + def add(request) + Typhoeus.before.each do |callback| + value = callback.call(request) + if value.nil? || value == false || value.is_a?(Response) + dequeue + return value + end + end + super + end + end + end +end diff --git a/lib/typhoeus/hydra/block_connection.rb b/lib/typhoeus/hydra/block_connection.rb new file mode 100644 index 0000000..9f3bd2f --- /dev/null +++ b/lib/typhoeus/hydra/block_connection.rb @@ -0,0 +1,35 @@ +module Typhoeus + class Hydra + + # This module handles the blocked connection request mode on + # the hydra side, where only stubbed requests + # are allowed. + # Connection blocking needs to be turned on: + # Typhoeus.configure do |config| + # config.block_connection = true + # end + # + # When trying to do real requests a NoStub error + # is raised. + # + # @api private + module BlockConnection + + # Overrides add in order to check before if block connection + # is turned on. If thats the case a NoStub error is + # raised. + # + # @example Add the request. + # hydra.add(request) + # + # @param [ Request ] request The request to enqueue. + def add(request) + if request.blocked? + raise Typhoeus::Errors::NoStub.new(request) + else + super + end + end + end + end +end diff --git a/lib/typhoeus/hydra/cacheable.rb b/lib/typhoeus/hydra/cacheable.rb new file mode 100644 index 0000000..74c7f30 --- /dev/null +++ b/lib/typhoeus/hydra/cacheable.rb @@ -0,0 +1,15 @@ +module Typhoeus + class Hydra + module Cacheable + def add(request) + if request.cacheable? && response = request.cached_response + response.cached = true + request.finish(response) + dequeue + else + super + end + end + end + end +end diff --git a/lib/typhoeus/hydra/memoizable.rb b/lib/typhoeus/hydra/memoizable.rb new file mode 100644 index 0000000..0402a86 --- /dev/null +++ b/lib/typhoeus/hydra/memoizable.rb @@ -0,0 +1,56 @@ +module Typhoeus + class Hydra + + # This module handles the GET request memoization + # on the hydra side. Memoization needs to be turned + # on: + # Typhoeus.configure do |config| + # config.memoize = true + # end + # + # @api private + module Memoizable + + # Return the memory. + # + # @example Return the memory. + # hydra.memory + # + # @return [ Hash ] The memory. + def memory + @memory ||= {} + end + + # Overrides add in order to check before if request + # is memoizable and already in memory. If thats the case, + # super is not called, instead the response is set and + # the on_complete callback called. + # + # @example Add the request. + # hydra.add(request) + # + # @param [ Request ] request The request to add. + # + # @return [ Request ] The added request. + def add(request) + if request.memoizable? && memory.has_key?(request) + response = memory[request] + request.finish(response, true) + dequeue + else + super + end + end + + # Overrides run to make sure the memory is cleared after + # each run. + # + # @example Run hydra. + # hydra.run + def run + super + memory.clear + end + end + end +end diff --git a/lib/typhoeus/hydra/queueable.rb b/lib/typhoeus/hydra/queueable.rb new file mode 100644 index 0000000..b53dc67 --- /dev/null +++ b/lib/typhoeus/hydra/queueable.rb @@ -0,0 +1,83 @@ +module Typhoeus + class Hydra + + # This module handles the request queueing on + # hydra. + # + # @api private + module Queueable + + # Return the queued requests. + # + # @example Return queued requests. + # hydra.queued_requests + # + # @return [ Array ] The queued requests. + def queued_requests + @queued_requests ||= [] + end + + # Abort the current hydra run as good as + # possible. This means that it only + # clears the queued requests and can't do + # anything about already running requests. + # + # @example Abort hydra. + # hydra.abort + def abort + queued_requests.clear + end + + # Enqueues a request in order to be performed + # by the hydra. This can even be done while + # the hydra is running. Also sets hydra on + # request. + # + # @example Queue request. + # hydra.queue(request) + def queue(request) + request.hydra = self + queued_requests << request + end + + # Pushes a request to the front of the queue, + # to be performed by the hydra. Also sets hydra + # on request + # + # @example Queue reques. + # hydra.queue_front(request) + def queue_front(request) + request.hydra = self + queued_requests.unshift request + end + + # Removes a request from queued_requests and + # adds it to the hydra in order to be + # performed next. + # + # @example Dequeue request. + # hydra.dequeue + # + # @since 0.6.4 + def dequeue + add(queued_requests.shift) unless queued_requests.empty? + end + + # Removes requests from queued_requests and + # adds them to the hydra until max_concurrency + # is reached. + # + # @example Dequeue requests. + # hydra.dequeue_many + # + # @since 0.6.8 + def dequeue_many + number = multi.easy_handles.count + until number == max_concurrency || queued_requests.empty? + add(queued_requests.shift) + number += 1 + end + end + end + end +end diff --git a/lib/typhoeus/hydra/runnable.rb b/lib/typhoeus/hydra/runnable.rb new file mode 100644 index 0000000..9c2ad51 --- /dev/null +++ b/lib/typhoeus/hydra/runnable.rb @@ -0,0 +1,19 @@ +module Typhoeus + class Hydra + + # This module contains logic to run a hydra. + module Runnable + + # Start the hydra run. + # + # @example Start hydra run. + # hydra.run + # + # @return [ Symbol ] Return value from multi.perform. + def run + dequeue_many + multi.perform + end + end + end +end diff --git a/lib/typhoeus/hydra/stubbable.rb b/lib/typhoeus/hydra/stubbable.rb new file mode 100644 index 0000000..930997c --- /dev/null +++ b/lib/typhoeus/hydra/stubbable.rb @@ -0,0 +1,28 @@ +module Typhoeus + class Hydra + + # This module handles stubbing on the hydra side. + # It plays well with the block_connection configuration, + # which raises when you make a request which is not stubbed. + # + # @api private + module Stubbable + + # Override add in order to check for matching expecations. + # When an expecation is found, super is not called. Instead a + # canned response is assigned to the request. + # + # @example Add the request. + # hydra.add(request) + def add(request) + if response = Expectation.response_for(request) + request.execute_headers_callbacks(response) + request.on_body.each{ |callback| callback.call(response.body, response) } + request.finish(response) + else + super + end + end + end + end +end diff --git a/lib/typhoeus/pool.rb b/lib/typhoeus/pool.rb new file mode 100644 index 0000000..552a73f --- /dev/null +++ b/lib/typhoeus/pool.rb @@ -0,0 +1,70 @@ +require 'thread' + +module Typhoeus + + # The easy pool stores already initialized + # easy handles for future use. This is useful + # because creating them is expensive. + # + # @api private + module Pool + @mutex = Mutex.new + @pid = Process.pid + + # Releases easy into the pool. The easy handle is + # reset before it gets back in. + # + # @example Release easy. + # Typhoeus::Pool.release(easy) + def self.release(easy) + easy.cookielist = "flush" # dump all known cookies to 'cookiejar' + easy.cookielist = "all" # remove all cookies from memory for this handle + easy.reset + @mutex.synchronize { easies << easy } + end + + # Return an easy from the pool. + # + # @example Return easy. + # Typhoeus::Pool.get + # + # @return [ Ethon::Easy ] The easy. + def self.get + @mutex.synchronize do + if @pid == Process.pid + easies.pop + else + # Process has forked. Clear all easies to avoid sockets being + # shared between processes. + @pid = Process.pid + easies.clear + nil + end + end || Ethon::Easy.new + end + + # Clear the pool + def self.clear + @mutex.synchronize { easies.clear } + end + + # Use yielded easy, will be released automatically afterwards. + # + # @example Use easy. + # Typhoeus::Pool.with_easy do |easy| + # # use easy + # end + def self.with_easy(&block) + easy = get + yield easy + ensure + release(easy) if easy + end + + private + + def self.easies + @easies ||= [] + end + end +end diff --git a/lib/typhoeus/railtie.rb b/lib/typhoeus/railtie.rb new file mode 100644 index 0000000..ee95f2a --- /dev/null +++ b/lib/typhoeus/railtie.rb @@ -0,0 +1,12 @@ +require "typhoeus" + +module Rails + module Typhoeus + class Railtie < Rails::Railtie + # Need to include the Typhoeus middleware. + initializer "include the identity map" do |app| + app.config.middleware.use "Rack::Typhoeus::Middleware::ParamsDecoder" + end + end + end +end diff --git a/lib/typhoeus/request.rb b/lib/typhoeus/request.rb new file mode 100644 index 0000000..c41ee7b --- /dev/null +++ b/lib/typhoeus/request.rb @@ -0,0 +1,221 @@ +require 'zlib' +require 'digest/sha1' +require 'typhoeus/request/actions' +require 'typhoeus/request/before' +require 'typhoeus/request/block_connection' +require 'typhoeus/request/cacheable' +require 'typhoeus/request/callbacks' +require 'typhoeus/request/marshal' +require 'typhoeus/request/memoizable' +require 'typhoeus/request/operations' +require 'typhoeus/request/responseable' +require 'typhoeus/request/streamable' +require 'typhoeus/request/stubbable' + +module Typhoeus + + # This class represents a request. + # + # @example (see #initialize) + # + # @example Make a request with the shortcut. + # response = Typhoeus.get("www.example.com") + # + # @see (see #initialize) + class Request + extend Request::Actions + include Request::Callbacks::Types + include Request::Callbacks + include Request::Streamable + include Request::Marshal + include Request::Operations + include Request::Responseable + include Request::Memoizable + include Request::Cacheable + include Request::BlockConnection + include Request::Stubbable + include Request::Before + + # Returns the provided base url. + # + # @return [ String ] + attr_accessor :base_url + + # Returns options, which includes default parameters. + # + # @return [ Hash ] + attr_accessor :options + + # Returns the hydra in which the request ran, if any. + # + # @return [ Typhoeus::Hydra ] + # + # @api private + attr_accessor :hydra + + # Returns the original options provided. + # + # @return [ Hash ] + # + # @api private + attr_accessor :original_options + + # @return [ Boolean ] + # + # @api private + attr_accessor :block_connection + + # Creates a new request. + # + # @example Simplest request. + # response = Typhoeus::Request.new("www.example.com").run + # + # @example Request with url parameters. + # response = Typhoeus::Request.new( + # "www.example.com", + # params: {a: 1} + # ).run + # + # @example Request with a body. + # response = Typhoeus::Request.new( + # "www.example.com", + # body: {b: 2} + # ).run + # + # @example Request with parameters and body. + # response = Typhoeus::Request.new( + # "www.example.com", + # params: {a: 1}, + # body: {b: 2} + # ).run + # + # @example Create a request and allow follow redirections. + # response = Typhoeus::Request.new( + # "www.example.com", + # followlocation: true + # ).run + # + # @param [ String ] base_url The url to request. + # @param [ options ] options The options. + # + # @option options [ Hash ] :params Translated + # into url parameters. + # @option options [ Hash ] :body Translated + # into HTTP POST request body. + # + # @return [ Typhoeus::Request ] The request. + # + # @note See {http://rubydoc.info/github/typhoeus/ethon/Ethon/Easy/Options Ethon::Easy::Options} for more options. + # + # @see Typhoeus::Hydra + # @see Typhoeus::Response + # @see Typhoeus::Request::Actions + def initialize(base_url, options = {}) + @base_url = base_url + @original_options = options + @options = options.dup + + set_defaults + end + + # Return the url. + # In contrast to base_url which returns the value you specified, url returns + # the full url including the parameters. + # + # @example Get the url. + # request.url + # + # @since 0.5.5 + def url + easy = EasyFactory.new(self).get + url = easy.url + Typhoeus::Pool.release(easy) + url + end + + # Returns whether other is equal to self. + # + # @example Are request equal? + # request.eql?(other_request) + # + # @param [ Object ] other The object to check. + # + # @return [ Boolean ] Returns true if equal, else false. + # + # @api private + def eql?(other) + self.class == other.class && + self.base_url == other.base_url && + fuzzy_hash_eql?(self.options, other.options) + end + + # Overrides Object#hash. + # + # @return [ Integer ] The integer representing the request. + # + # @api private + def hash + Zlib.crc32 cache_key + end + + # Returns a cache key for use with caching methods that required a string + # for a key. Will get used by ActiveSupport::Cache stores automatically. + # + # @return [ String ] The cache key. + def cache_key + Digest::SHA1.hexdigest "#{self.class.name}#{base_url}#{hashable_string_for(options)}" + end + + # Mimics libcurls POST body generation. This is not accurate, but good + # enough for VCR. + # + # @return [ String ] The encoded body. + # otherwise. + # + # @api private + def encoded_body + Ethon::Easy::Form.new(nil, options[:body]).to_s + end + + private + + # Checks if two hashes are equal or not, discarding + # first-level hash order. + # + # @param [ Hash ] left + # @param [ Hash ] right hash to check for equality + # + # @return [ Boolean ] Returns true if hashes have + # same values for same keys and same length, + # even if the keys are given in a different order. + def fuzzy_hash_eql?(left, right) + return true if (left == right) + + (left.count == right.count) && left.inject(true) do |res, kvp| + res && (kvp[1] == right[kvp[0]]) + end + end + + def hashable_string_for(obj) + case obj + when Hash + hashable_string_for(obj.sort_by {|sub_obj| sub_obj.first.to_s}) + when Array + obj.map {|sub_obj| hashable_string_for(sub_obj)}.to_s + else + obj.to_s + end + end + + # Sets default header and verbose when turned on. + def set_defaults + default_user_agent = Config.user_agent || Typhoeus::USER_AGENT + + options[:headers] = {'User-Agent' => default_user_agent}.merge(options[:headers] || {}) + options[:headers]['Expect'] ||= '' + options[:verbose] = Typhoeus::Config.verbose if options[:verbose].nil? && !Typhoeus::Config.verbose.nil? + options[:maxredirs] ||= 50 + options[:proxy] = Typhoeus::Config.proxy unless options.has_key?(:proxy) || Typhoeus::Config.proxy.nil? + end + end +end diff --git a/lib/typhoeus/request/actions.rb b/lib/typhoeus/request/actions.rb new file mode 100644 index 0000000..b7974d8 --- /dev/null +++ b/lib/typhoeus/request/actions.rb @@ -0,0 +1,125 @@ +module Typhoeus + class Request + + # Module containing logic about shortcuts to + # http methods. Like + # Typhoeus.get("www.example.com") + module Actions + + # Make a get request. + # + # @example Make get request. + # Typhoeus.get("www.example.com") + # + # @param (see Typhoeus::Request#initialize) + # + # @option (see Typhoeus::Request#initialize) + # + # @return (see Typhoeus::Response#initialize) + # + # @note (see Typhoeus::Request#initialize) + def get(base_url, options = {}) + Request.new(base_url, options.merge(:method => :get)).run + end + + # Make a post request. + # + # @example Make post request. + # Typhoeus.post("www.example.com") + # + # @param (see Typhoeus::Request#initialize) + # + # @option (see Typhoeus::Request#initialize) + # + # @return (see Typhoeus::Response#initialize) + # + # @note (see Typhoeus::Request#initialize) + def post(base_url, options = {}) + Request.new(base_url, options.merge(:method => :post)).run + end + + # Make a put request. + # + # @example Make put request. + # Typhoeus.put("www.example.com") + # + # @param (see Typhoeus::Request#initialize) + # + # @option options :params [ Hash ] Params hash which + # is attached to the base_url. + # @option options :body [ Hash ] Body hash which + # becomes a PUT request body. + # + # @return (see Typhoeus::Response#initialize) + # + # @note (see Typhoeus::Request#initialize) + def put(base_url, options = {}) + Request.new(base_url, options.merge(:method => :put)).run + end + + # Make a delete request. + # + # @example Make delete request. + # Typhoeus.delete("www.example.com") + # + # @param (see Typhoeus::Request#initialize) + # + # @option (see Typhoeus::Request#initialize) + # + # @return (see Typhoeus::Response#initialize) + # + # @note (see Typhoeus::Request#initialize) + def delete(base_url, options = {}) + Request.new(base_url, options.merge(:method => :delete)).run + end + + # Make a head request. + # + # @example Make head request. + # Typhoeus.head("www.example.com") + # + # @param (see Typhoeus::Request#initialize) + # + # @option (see Typhoeus::Request#initialize) + # + # @return (see Typhoeus::Response#initialize) + # + # @note (see Typhoeus::Request#initialize) + def head(base_url, options = {}) + Request.new(base_url, options.merge(:method => :head)).run + end + + # Make a patch request. + # + # @example Make patch request. + # Typhoeus.patch("www.example.com") + # + # @param (see Typhoeus::Request#initialize) + # + # @option (see Typhoeus::Request#initialize) + # + # @return (see Typhoeus::Response#initialize) + # + # @note (see Typhoeus::Request#initialize) + def patch(base_url, options = {}) + Request.new(base_url, options.merge(:method => :patch)).run + end + + # Make a options request. + # + # @example Make options request. + # Typhoeus.options("www.example.com") + # + # @param (see Typhoeus::Request#initialize) + # + # @option (see Typhoeus::Request#initialize) + # + # @return (see Typhoeus::Response#initialize) + # + # @note (see Typhoeus::Request#initialize) + def options(base_url, options = {}) + Request.new(base_url, options.merge(:method => :options)).run + end + end + end +end diff --git a/lib/typhoeus/request/before.rb b/lib/typhoeus/request/before.rb new file mode 100644 index 0000000..d6ad765 --- /dev/null +++ b/lib/typhoeus/request/before.rb @@ -0,0 +1,30 @@ +module Typhoeus + class Request + + # This module provides a way to hook into before + # a request runs. This is very powerful + # and you should be careful because when you accidently + # return a falsy value the request won't be executed. + # + # @api private + module Before + + # Overrride run in order to execute callbacks in + # Typhoeus.before. Will break and return when a + # callback returns nil or false. Calls super + # otherwise. + # + # @example Run the request. + # request.run + def run + Typhoeus.before.each do |callback| + value = callback.call(self) + if value.nil? || value == false || value.is_a?(Response) + return response + end + end + super + end + end + end +end diff --git a/lib/typhoeus/request/block_connection.rb b/lib/typhoeus/request/block_connection.rb new file mode 100644 index 0000000..7fc966b --- /dev/null +++ b/lib/typhoeus/request/block_connection.rb @@ -0,0 +1,52 @@ +module Typhoeus + class Request + + # This module handles the blocked connection request mode on + # the request side, where only stubbed requests + # are allowed. + # Connection blocking needs to be turned on: + # Typhoeus.configure do |config| + # config.block_connection = true + # end + # + # When trying to do real requests a NoStub error + # is raised. + # + # @api private + module BlockConnection + + # Overrides run in order to check before if block connection + # is turned on. If thats the case a NoStub error is + # raised. + # + # @example Run request. + # request.run + # + # @raise [Typhoeus::Errors::NoStub] If connection is blocked + # and no stub defined. + def run + if blocked? + raise Typhoeus::Errors::NoStub.new(self) + else + super + end + end + + # Returns wether a request is blocked or not. Takes + # request.block_connection and Typhoeus::Config.block_connection + # into consideration. + # + # @example Blocked? + # request.blocked? + # + # @return [ Boolean ] True if blocked, false else. + def blocked? + if block_connection.nil? + Typhoeus::Config.block_connection + else + block_connection + end + end + end + end +end diff --git a/lib/typhoeus/request/cacheable.rb b/lib/typhoeus/request/cacheable.rb new file mode 100644 index 0000000..74a41a9 --- /dev/null +++ b/lib/typhoeus/request/cacheable.rb @@ -0,0 +1,38 @@ +module Typhoeus + class Request + module Cacheable + def response=(response) + cache.set(self, response) if cacheable? && !response.cached? + super + end + + def cacheable? + cache + end + + def run + if response = cached_response + response.cached = true + finish(response) + else + super + end + end + + def cached_response + cacheable? && cache.get(self) + end + + def cache_ttl + options[:cache_ttl] + end + + private + + def cache + return nil if options[:cache] === false + options[:cache] || Typhoeus::Config.cache + end + end + end +end diff --git a/lib/typhoeus/request/callbacks.rb b/lib/typhoeus/request/callbacks.rb new file mode 100644 index 0000000..cc2cc1e --- /dev/null +++ b/lib/typhoeus/request/callbacks.rb @@ -0,0 +1,151 @@ +module Typhoeus + class Request + + # This module contains the logic for the response callbacks. + # + # You can set multiple callbacks, which are then executed + # in the same order. + # + # request.on_complete { |response| p 1 } + # request.on_complete { |response| p 2 } + # request.execute_callbacks + # #=> 1 + # #=> 2 + # + # You can clear the callbacks: + # + # request.on_complete { |response| p 1 } + # request.on_complete { |response| p 2 } + # request.on_complete.clear + # request.execute_callbacks + # #=> nil + # + # @note If you're using the Hydra to execute multiple + # requests, then callbacks are delaying the + # request execution. + module Callbacks + + module Types # :nodoc: + # Set on_complete callback. + # + # @example Set on_complete. + # request.on_complete { |response| p "yay" } + # + # @param [ Block ] block The block to execute. + # + # @yield [ Typhoeus::Response ] + # + # @return [ Array ] All on_complete blocks. + def on_complete(&block) + @on_complete ||= [] + @on_complete << block if block_given? + @on_complete + end + + # Set on_success callback. + # + # @example Set on_success. + # request.on_success { |response| p "yay" } + # + # @param [ Block ] block The block to execute. + # + # @yield [ Typhoeus::Response ] + # + # @return [ Array ] All on_success blocks. + def on_success(&block) + @on_success ||= [] + @on_success << block if block_given? + @on_success + end + + # Set on_failure callback. + # + # @example Set on_failure. + # request.on_failure { |response| p "yay" } + # + # @param [ Block ] block The block to execute. + # + # @yield [ Typhoeus::Response ] + # + # @return [ Array ] All on_failure blocks. + def on_failure(&block) + @on_failure ||= [] + @on_failure << block if block_given? + @on_failure + end + + # Set on_headers callback. + # + # @example Set on_headers. + # request.on_headers { |response| p "yay" } + # + # @param [ Block ] block The block to execute. + # + # @yield [ Typhoeus::Response ] + # + # @return [ Array ] All on_headers blocks. + def on_headers(&block) + @on_headers ||= [] + @on_headers << block if block_given? + @on_headers + end + + # Set on_progress callback. + # + # @example Set on_progress. + # request.on_progress do |dltotal, dlnow, ultotal, ulnow| + # puts "dltotal (#{dltotal}), dlnow (#{dlnow}), ultotal (#{ultotal}), ulnow (#{ulnow})" + # end + # + # @param [ Block ] block The block to execute. + # + # @yield [ Typhoeus::Response ] + # + # @return [ Array ] All on_progress blocks. + def on_progress(&block) + @on_progress ||= [] + @on_progress << block if block_given? + @on_progress + end + end + + # Execute the headers callbacks and yields response. + # + # @example Execute callbacks. + # request.execute_headers_callbacks + # + # @return [ Array ] The results of the on_headers callbacks. + # + # @api private + def execute_headers_callbacks(response) + (Typhoeus.on_headers + on_headers).map do |callback| + callback.call(response) + end + end + + # Execute necessary callback and yields response. This + # include in every case on_complete and on_progress, on_success + # if successful and on_failure if not. + # + # @example Execute callbacks. + # request.execute_callbacks + # + # @return [ void ] + # + # @api private + def execute_callbacks + callbacks = Typhoeus.on_complete + Typhoeus.on_progress + on_complete + on_progress + + if response && response.success? + callbacks += Typhoeus.on_success + on_success + elsif response + callbacks += Typhoeus.on_failure + on_failure + end + + callbacks.each do |callback| + self.response.handled_response = callback.call(self.response) + end + end + end + end +end diff --git a/lib/typhoeus/request/marshal.rb b/lib/typhoeus/request/marshal.rb new file mode 100644 index 0000000..3c8387c --- /dev/null +++ b/lib/typhoeus/request/marshal.rb @@ -0,0 +1,22 @@ +module Typhoeus + class Request + + # This module contains custom serializer. + module Marshal + + # Return the important data needed to serialize this Request, except the + # request callbacks and `hydra`, since they cannot be marshalled. + def marshal_dump + unmarshallable = %w(@on_complete @on_success @on_failure @on_progress @on_headers @on_body @hydra) + (instance_variables - unmarshallable - unmarshallable.map(&:to_sym)).map do |name| + [name, instance_variable_get(name)] + end + end + + # Load. + def marshal_load(attributes) + attributes.each { |name, value| instance_variable_set(name, value) } + end + end + end +end diff --git a/lib/typhoeus/request/memoizable.rb b/lib/typhoeus/request/memoizable.rb new file mode 100644 index 0000000..c09ab10 --- /dev/null +++ b/lib/typhoeus/request/memoizable.rb @@ -0,0 +1,38 @@ +module Typhoeus + class Request + + # This module handles the GET request memoization + # on the request side. Memoization needs to be turned + # on: + # Typhoeus.configure do |config| + # config.memoize = true + # end + # + # @api private + module Memoizable + + # Override response setter and memoizes response + # if the request is memoizable. + # + # @param [ Response ] response The response to set. + # + # @example Set response. + # request.response = response + def response=(response) + hydra.memory[self] = response if memoizable? + super + end + + # Return whether a request is memoizable. + # + # @example Is request memoizable? + # request.memoizable? + # + # @return [ Boolean ] Return true if memoizable, false else. + def memoizable? + Typhoeus::Config.memoize && + (options[:method].nil? || options[:method] == :get) + end + end + end +end diff --git a/lib/typhoeus/request/operations.rb b/lib/typhoeus/request/operations.rb new file mode 100644 index 0000000..a461aa4 --- /dev/null +++ b/lib/typhoeus/request/operations.rb @@ -0,0 +1,40 @@ +module Typhoeus + class Request + + # This module contains everything what is necessary + # to make a single request. + module Operations + + # Run a request. + # + # @example Run a request. + # Typhoeus::Request.new("www.example.com").run + # + # @return [ Response ] The response. + def run + easy = EasyFactory.new(self).get + easy.perform + response + end + + # Sets a response, the request on the response + # and executes the callbacks. + # + # @param [Typhoeus::Response] response The response. + # @param [Boolean] bypass_memoization Wether to bypass + # memoization or not. Decides how the response is set. + # + # @return [Typhoeus::Response] The response. + def finish(response, bypass_memoization = nil) + if bypass_memoization + @response = response + else + self.response = response + end + self.response.request = self + execute_callbacks + response + end + end + end +end diff --git a/lib/typhoeus/request/responseable.rb b/lib/typhoeus/request/responseable.rb new file mode 100644 index 0000000..2b91a6d --- /dev/null +++ b/lib/typhoeus/request/responseable.rb @@ -0,0 +1,29 @@ +module Typhoeus + class Request + + # This module contains logic for having a reponse + # getter and setter. + module Responseable + + # Set the response. + # + # @example Set response. + # request.response = response + # + # @param [ Response ] value The response to set. + def response=(value) + @response = value + end + + # Return the response. + # + # @example Return response. + # request.response + # + # @return [ Response ] The response. + def response + @response ||= nil + end + end + end +end diff --git a/lib/typhoeus/request/streamable.rb b/lib/typhoeus/request/streamable.rb new file mode 100644 index 0000000..0cfaa48 --- /dev/null +++ b/lib/typhoeus/request/streamable.rb @@ -0,0 +1,34 @@ +module Typhoeus + class Request + + # This module contians the logic for response streaming. + module Streamable + + # Set on_body callback. + # + # This callback will be called each time a portion of the body is read from the socket. + # Setting an on_body callback will cause the response body to be empty. + # + # @example Set on_body. + # request.on_body { |body_chunk, response| puts "Got #{body_chunk.bytesize} bytes" } + # + # @param [ Block ] block The block to execute. + # + # @yield [ Typhoeus::Response, String ] + # + # @return [ Array ] All on_body blocks. + def on_body(&block) + @on_body ||= [] + @on_body << block if block_given? + @on_body + end + + # Is this request using streaming? + # + # @return [ Boolean ] True if any on_body blocks have been set. + def streaming? + defined?(@on_body) && @on_body.any? + end + end + end +end diff --git a/lib/typhoeus/request/stubbable.rb b/lib/typhoeus/request/stubbable.rb new file mode 100644 index 0000000..ee42557 --- /dev/null +++ b/lib/typhoeus/request/stubbable.rb @@ -0,0 +1,30 @@ +module Typhoeus + class Request + + # This module handles stubbing on the request side. + # It plays well with the block_connection configuration, + # which raises when you make a request which is not stubbed. + # + # @api private + module Stubbable + + # Override run in order to check for matching expectations. + # When an expectation is found, super is not called. Instead a + # canned response is assigned to the request. + # + # @example Run the request. + # request.run + # + # @return [ Response ] The response. + def run + if response = Expectation.response_for(self) + execute_headers_callbacks(response) + self.on_body.each{ |callback| callback.call(response.body, response) } + finish(response) + else + super + end + end + end + end +end diff --git a/lib/typhoeus/response.rb b/lib/typhoeus/response.rb new file mode 100644 index 0000000..efa7dbd --- /dev/null +++ b/lib/typhoeus/response.rb @@ -0,0 +1,68 @@ +require 'typhoeus/response/header' +require 'typhoeus/response/informations' +require 'typhoeus/response/status' +require 'typhoeus/response/cacheable' + +module Typhoeus + + # This class represents the response. + class Response + include Response::Informations + include Response::Status + include Response::Cacheable + + # Remembers the corresponding request. + # + # @example Get request. + # request = Typhoeus::Request.new("www.example.com") + # response = request.run + # request == response.request + # #=> true + # + # @return [ Typhoeus::Request ] + attr_accessor :request + + # The provided options, which contain all the + # informations about the request. + # + # @return [ Hash ] + attr_accessor :options + + # Set the handled response. + attr_writer :handled_response + + # @api private + attr_writer :mock + + # Create a new response. + # + # @example Create a response. + # Response.new + # + # @param [ Hash ] options The options hash. + # + # @return [ Response ] The new response. + def initialize(options = {}) + @options = options + @headers = Header.new(options[:headers]) if options[:headers] + end + + # Returns whether this request is mocked + # or not. + # + # @api private + def mock + defined?(@mock) ? @mock : options[:mock] + end + alias :mock? :mock + + # Returns the handled_response if it has + # been defined; otherwise, returns the response + # + # @return [ Object ] The result of callbacks + # done on the response or the original response. + def handled_response + @handled_response || self + end + end +end diff --git a/lib/typhoeus/response/cacheable.rb b/lib/typhoeus/response/cacheable.rb new file mode 100644 index 0000000..0170171 --- /dev/null +++ b/lib/typhoeus/response/cacheable.rb @@ -0,0 +1,14 @@ +module Typhoeus + class Response + module Cacheable + + # Set the cache status, if we got response from cache + # it will have cached? == true + attr_writer :cached + + def cached? + defined?(@cached) ? !!@cached : false + end + end + end +end diff --git a/lib/typhoeus/response/header.rb b/lib/typhoeus/response/header.rb new file mode 100644 index 0000000..11580ab --- /dev/null +++ b/lib/typhoeus/response/header.rb @@ -0,0 +1,105 @@ +require 'delegate' + +module Typhoeus + class Response + + # This class represents the response header. + # It can be accessed like a hash. + # Values can be strings (normal case) or arrays of strings (for duplicates headers) + # + # @api private + class Header < DelegateClass(Hash) + + # Create a new header. + # + # @example Create new header. + # Header.new(raw) + # + # @param [ String ] raw The raw header. + def initialize(raw) + super({}) + @raw = raw + @sanitized = {} + parse + end + + def [](key) + fetch(key) { @sanitized[key.to_s.downcase] } + end + + # Parses the raw header. + # + # @example Parse header. + # header.parse + def parse + case @raw + when Hash + raw.each do |k, v| + process_pair(k, v) + end + when String + raw.split(/\r?\n(?!\s)/).each do |header| + header.strip! + next if header.empty? || header.start_with?( 'HTTP/' ) + process_line(header) + end + end + end + + private + + # Processes line and saves the result. + # + # @return [ void ] + def process_line(header) + key, value = header.split(':', 2) + process_pair(key.strip, (value ? value.strip.gsub(/\r?\n\s*/, ' ') : '')) + end + + # Sets key value pair for self and @sanitized. + # + # @return [ void ] + def process_pair(key, value) + set_value(key, value, self) + @sanitized[key.downcase] = self[key] + end + + # Sets value for key in specified hash + # + # @return [ void ] + def set_value(key, value, hash) + current_value = hash[key] + if current_value + if current_value.is_a? Array + current_value << value + else + hash[key] = [current_value, value] + end + else + hash[key] = value + end + end + + # Returns the raw header or empty string. + # + # @example Return raw header. + # header.raw + # + # @return [ String ] The raw header. + def raw + @raw || '' + end + + # Sets the default proc for the specified hash independent of the Ruby version. + # + # @return [ void ] + def set_default_proc_on(hash, default_proc) + if hash.respond_to?(:default_proc=) + hash.default_proc = default_proc + else + hash.replace(Hash.new(&default_proc).merge(hash)) + end + end + end + end +end diff --git a/lib/typhoeus/response/informations.rb b/lib/typhoeus/response/informations.rb new file mode 100644 index 0000000..73544dd --- /dev/null +++ b/lib/typhoeus/response/informations.rb @@ -0,0 +1,248 @@ +module Typhoeus + class Response + + # This module contains logic about informations + # on a response. + module Informations + + # Return libcurls return value. + # + # @example Get return_code. + # response.return_code + # + # @return [ Symbol ] The return_code. + def return_code + options[:return_code] + end + + # Returns a string describing the return. + # + # @example Get return_message. + # response.return_message + # + # @return [ String ] The return_message. + # + # @since 0.6.2 + def return_message + Ethon::Curl.easy_strerror(return_code) if return_code + end + + # Return the http response body. + # + # @example Get response_body. + # response.response_body + # + # @return [ String ] The response_body. + def response_body + options[:response_body] || options[:body] + end + alias :body :response_body + + # Return the http response headers. + # + # @example Get response_headers. + # response.response_headers + # + # @return [ String ] The response_headers. + def response_headers + return options[:response_headers] if options[:response_headers] + if mock? && h = options[:headers] + status_code = return_code || "200" + reason_phrase = status_code == "200" ? "OK" : "Mock Reason Phrase" + status_line = "HTTP/1.1 #{status_code} #{reason_phrase}" + actual_headers = h.map{ |k,v| [k, v.respond_to?(:join) ? v.join(',') : v] }. + map{ |e| "#{e.first}: #{e.last}" } + + [status_line, *actual_headers].join("\r\n") + end + end + + # Return the last received HTTP, FTP or SMTP response code. + # The value will be zero if no server response code has + # been received. Note that a proxy's CONNECT response should + # be read with http_connect_code and not this. + # + # @example Get response_code. + # response.response_code + # + # @return [ Integer ] The response_code. + def response_code + (options[:response_code] || options[:code]).to_i + end + alias :code :response_code + + # Return the available http auth methods. + # Bitmask indicating the authentication method(s) + # available. + # + # @example Get httpauth_avail. + # response.httpauth_avail + # + # @return [ Integer ] The bitmask. + def httpauth_avail + options[:httpauth_avail] + end + + + # Return the total time in seconds for the previous + # transfer, including name resolving, TCP connect etc. + # + # @example Get total_time. + # response.total_time + # + # @return [ Float ] The total_time. + def total_time + options[:total_time] || options[:time] + end + alias :time :total_time + + # Return the time, in seconds, it took from the start + # until the first byte is received by libcurl. This + # includes pretransfer time and also the time the + # server needs to calculate the result. + # + # @example Get starttransfer_time. + # response.starttransfer_time + # + # @return [ Float ] The starttransfer_time. + def starttransfer_time + options[:starttransfer_time] || options[:start_transfer_time] + end + alias :start_transfer_time :starttransfer_time + + # Return the time, in seconds, it took from the start + # until the SSL/SSH connect/handshake to the remote + # host was completed. This time is most often very near + # to the pre transfer time, except for cases such as HTTP + # pipelining where the pretransfer time can be delayed + # due to waits in line for the pipeline and more. + # + # @example Get appconnect_time. + # response.appconnect_time + # + # @return [ Float ] The appconnect_time. + def appconnect_time + options[:appconnect_time] || options[:app_connect_time] + end + alias :app_connect_time :appconnect_time + + # Return the time, in seconds, it took from the start + # until the file transfer is just about to begin. This + # includes all pre-transfer commands and negotiations + # that are specific to the particular protocol(s) involved. + # It does not involve the sending of the protocol- + # specific request that triggers a transfer. + # + # @example Get pretransfer_time. + # response.pretransfer_time + # + # @return [ Float ] The pretransfer_time. + def pretransfer_time + options[:pretransfer_time] + end + + # Return the time, in seconds, it took from the start + # until the connect to the remote host (or proxy) was completed. + # + # @example Get connect_time. + # response.connect_time + # + # @return [ Float ] The connect_time. + def connect_time + options[:connect_time] + end + + # Return the time, in seconds, it took from the + # start until the name resolving was completed. + # + # @example Get namelookup_time. + # response.namelookup_time + # + # @return [ Float ] The namelookup_time. + def namelookup_time + options[:namelookup_time] || options[:name_lookup_time] + end + alias :name_lookup_time :namelookup_time + + # Return the time, in seconds, it took for all redirection steps + # include name lookup, connect, pretransfer and transfer before the + # final transaction was started. time_redirect shows the complete + # execution time for multiple redirections. + # + # @example Get redirect_time. + # response.redirect_time + # + # @return [ Float ] The redirect_time. + def redirect_time + options[:redirect_time] + end + + # Return the last used effective url. + # + # @example Get effective_url. + # response.effective_url + # + # @return [ String ] The effective_url. + def effective_url + options[:effective_url] + end + + # Return the string holding the IP address of the most recent + # connection done with this curl handle. This string + # may be IPv6 if that's enabled. + # + # @example Get primary_ip. + # response.primary_ip + # + # @return [ String ] The primary_ip. + def primary_ip + options[:primary_ip] + end + + # Return the total number of redirections that were + # actually followed + # + # @example Get redirect_count. + # response.redirect_count + # + # @return [ Integer ] The redirect_count. + def redirect_count + options[:redirect_count] + end + + def request_size + options[:request_size] + end + + def debug_info + options[:debug_info] + end + + # Returns the response header. + # + # @example Return headers. + # response.headers + # + # @return [ Typhoeus::Header ] The response header. + def headers + return Header.new(options[:headers]) if mock? && options[:headers] + return nil if response_headers.nil? && !defined?(@headers) + @headers ||= Header.new(response_headers.split("\r\n\r\n").last) + end + alias :headers_hash :headers + + # Return all redirections in between as multiple + # responses with header. + # + # @example Return redirections. + # response.redirections + # + # @return [ Array ] The redirections + def redirections + return [] unless response_headers + response_headers.split("\r\n\r\n")[0..-2].map{ |h| Response.new(:response_headers => h) } + end + end + end +end + diff --git a/lib/typhoeus/response/status.rb b/lib/typhoeus/response/status.rb new file mode 100644 index 0000000..e1840f4 --- /dev/null +++ b/lib/typhoeus/response/status.rb @@ -0,0 +1,106 @@ +module Typhoeus + class Response + + # This module contains logic about the http + # status. + module Status + + # Return the status message if present. + # + # @example Return status message. + # reesponse.status_message + # + # @return [ String ] The message. + def status_message + return @status_message if defined?(@status_message) && @status_message + return options[:status_message] unless options[:status_message].nil? + + # HTTP servers can choose not to include the explanation to HTTP codes. The RFC + # states this (http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html#sec10.4): + # Except when responding to a HEAD request, the server SHOULD include an entity containing + # an explanation of the error situation [...] + # This means 'HTTP/1.1 404' is as valid as 'HTTP/1.1 404 Not Found' and we have to handle it. + # + # Regexp doc: http://rubular.com/r/eAr1oVYsVa + if first_header_line != nil and first_header_line[/\d{3} (.*)$/, 1] != nil + @status_message = first_header_line[/\d{3} (.*)$/, 1].chomp + else + @status_message = nil + end + end + + # Return the http version. + # + # @example Return http version. + # response.http_version + # + # @return [ String ] The http version. + def http_version + @http_version ||= first_header_line ? first_header_line[/HTTP\/(\S+)/, 1] : nil + end + + # Return whether the response is a success. + # + # @example Return if the response was successful. + # response.success? + # + # @return [ Boolean ] Return true if successful, false else. + def success? + (mock || return_code == :ok) && response_code && has_good_response_code? + end + + # Return whether the response is a failure. + # + # @example Return if the response was failed. + # response.failure? + # + # @return [ Boolean ] Return true if failure, false else. + def failure? + (mock || return_code == :internal_server_error) && response_code && has_bad_response_code? + end + + # Return wether the response is modified. + # + # @example Return if the response was modified. + # response.modified? + # + # @return [ Boolean ] Return true if modified, false else. + def modified? + (mock || return_code == :ok) && response_code && response_code != 304 + end + + # Return whether the response is timed out. + # + # @example Return if the response timed out. + # response.timed_out? + # + # @return [ Boolean ] Return true if timed out, false else. + def timed_out? + return_code == :operation_timedout + end + + private + + # :nodoc: + def first_header_line + @first_header_line ||= begin + if response_headers.to_s.include?("\r\n\r\n") + response_headers.to_s.split("\r\n\r\n").last.split("\r\n").first + else + response_headers.to_s.split("\r\n").first + end + end + end + + # :nodoc: + def has_good_response_code? + response_code >= 200 && response_code < 300 + end + + # :nodoc: + def has_bad_response_code? + !has_good_response_code? + end + end + end +end diff --git a/lib/typhoeus/version.rb b/lib/typhoeus/version.rb new file mode 100644 index 0000000..048f091 --- /dev/null +++ b/lib/typhoeus/version.rb @@ -0,0 +1,5 @@ +module Typhoeus + + # The current Typhoeus version. + VERSION = '1.4.0' +end diff --git a/perf/profile.rb b/perf/profile.rb new file mode 100644 index 0000000..29ec9c2 --- /dev/null +++ b/perf/profile.rb @@ -0,0 +1,14 @@ +require 'typhoeus' +require 'ruby-prof' + +calls = 50 +base_url = "http://127.0.0.1:3000/" + +RubyProf.start +calls.times do |i| + Typhoeus::Request.get(base_url+i.to_s) +end +result = RubyProf.stop + +printer = RubyProf::FlatPrinter.new(result) +printer.print(STDOUT) diff --git a/perf/vs_nethttp.rb b/perf/vs_nethttp.rb new file mode 100644 index 0000000..305f9fe --- /dev/null +++ b/perf/vs_nethttp.rb @@ -0,0 +1,64 @@ +require 'typhoeus' +require 'net/http' +require 'open-uri' +require 'benchmark' + +URL = "http://localhost:300" +hydra = Typhoeus::Hydra.new(max_concurrency: 3) + +if defined? require_relative + require_relative '../spec/support/localhost_server.rb' + require_relative '../spec/support/server.rb' +else + require '../spec/support/localhost_server.rb' + require '../spec/support/server.rb' +end +LocalhostServer.new(TESTSERVER.new, 3000) +LocalhostServer.new(TESTSERVER.new, 3001) +LocalhostServer.new(TESTSERVER.new, 3002) + +def url_for(i) + "#{URL}#{i%3}/" +end + +Benchmark.bm do |bm| + + [1000].each do |calls| + puts "[ #{calls} requests ]" + + bm.report("net/http ") do + calls.times do |i| + uri = URI.parse(url_for(i)) + Net::HTTP.get_response(uri) + end + end + + bm.report("open ") do + calls.times do |i| + open(url_for(i)) + end + end + + bm.report("request ") do + calls.times do |i| + Typhoeus::Request.get(url_for(i)) + end + end + + bm.report("hydra ") do + calls.times do |i| + hydra.queue(Typhoeus::Request.new(url_for(i))) + end + hydra.run + end + + bm.report("hydra memoize ") do + Typhoeus::Config.memoize = true + calls.times do |i| + hydra.queue(Typhoeus::Request.new(url_for(i))) + end + hydra.run + Typhoeus::Config.memoize = false + end + end +end diff --git a/spec/rack/typhoeus/middleware/params_decoder/helper_spec.rb b/spec/rack/typhoeus/middleware/params_decoder/helper_spec.rb new file mode 100644 index 0000000..3aee759 --- /dev/null +++ b/spec/rack/typhoeus/middleware/params_decoder/helper_spec.rb @@ -0,0 +1,156 @@ +require 'spec_helper' +require "rack/typhoeus" + +describe "Rack::Typhoeus::Middleware::ParamsDecoder::Helper" do + + let(:klass) do + Class.new do + include Rack::Typhoeus::Middleware::ParamsDecoder::Helper + end.new + end + + describe "#decode" do + let(:decoded) { klass.decode(params) } + let(:params) { { :array => {'0' => :a, '1' => :b } } } + + it "decodes" do + expect(decoded[:array]).to match_array([:a, :b]) + end + + it "doesn't modify" do + expect(decoded).to_not be(params) + end + end + + describe "#decode!" do + let(:decoded) { klass.decode!(params) } + + context "when hash" do + context "when encoded" do + context "when simple" do + let(:params) { { :array => {'0' => :a, '1' => :b } } } + + it "decodes" do + expect(decoded[:array]).to match_array([:a, :b]) + end + + it "modifies" do + expect(decoded).to eq(params) + end + end + + context "when longer and more complex" do + let(:params) do + { + :ids => { + "0" => "407304", + "1" => "407305", + "2" => "407306", + "3" => "407307", + "4" => "407308", + "5" => "407309", + "6" => "407310", + "7" => "407311", + "8" => "407312", + "9" => "407313", + "10" => "327012" + } + } + end + + it "decodes ensuring arrays maintain their original order" do + expect(decoded[:ids]).to eq(["407304", "407305", "407306", "407307", "407308", "407309", "407310", "407311", "407312", "407313", "327012"]) + end + end + + context "when nested" do + let(:params) do + { :array => { '0' => 0, '1' => { '0' => 'sub0', '1' => 'sub1' } } } + end + + it "decodes" do + expect(decoded[:array]).to include(0) + expect(decoded[:array].find{|e| e.is_a?(Array)}).to( + match_array(['sub0', 'sub1']) + ) + end + + it "modifies" do + expect(decoded).to eq(params) + end + end + end + + context "when not encoded" do + let(:params) { {:a => :a} } + + it "doesn't modify" do + expect(decoded).to be(params) + end + end + end + + context "when no hash" do + let(:params) { "a" } + + it "returns self" do + expect(decoded).to be(params) + end + end + end + + describe "#encoded?" do + let(:encoded) { klass.send(:encoded?, params) } + + context "when there is only one key" do + context "and its 0" do + let(:params){ {'0' => 1} } + it 'returns true' do + expect(encoded).to be_truthy + end + end + context "and its not 0" do + let(:params){ {'some-key' => 1}} + it 'returns false' do + expect(encoded).to be_falsey + end + end + end + + context "when keys are ascending numbers starting with zero" do + let(:params) { Hash[12.times.map {|i| [i, (i+65).chr]}] } + + it "returns true" do + expect(encoded).to be_truthy + end + end + + context "when keys are not ascending numbers starting with zero" do + let(:params) { {:a => 1} } + + it "returns false" do + expect(encoded).to be_falsey + end + end + end + + describe "#convert" do + let(:converted) { klass.send(:convert, params) } + + context "when encoded" do + let(:params) { {'0' => :a, '1' => :b} } + + it "returns values" do + expect(converted).to match_array([:a, :b]) + end + end + + context "when not encoded" do + let(:params) { {:a => :a} } + + it "returns unmodified" do + expect(converted).to be(params) + end + end + end +end diff --git a/spec/rack/typhoeus/middleware/params_decoder_spec.rb b/spec/rack/typhoeus/middleware/params_decoder_spec.rb new file mode 100644 index 0000000..187da9f --- /dev/null +++ b/spec/rack/typhoeus/middleware/params_decoder_spec.rb @@ -0,0 +1,31 @@ +require 'spec_helper' + +describe "Rack::Typhoeus::Middleware::ParamsDecoder" do + + before(:all) do + require "rack/typhoeus" + end + + let(:app) do + double + end + + let(:env) do + double + end + + let(:klass) do + Rack::Typhoeus::Middleware::ParamsDecoder + end + + describe "#call" do + end + + context "when requesting" do + let(:response) { Typhoeus.get("localhost:3001", :params => {:x => [:a]}) } + + it "transforms parameters" do + expect(response.body).to include("query_hash\":{\"x\":[\"a\"]}") + end + end +end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb new file mode 100644 index 0000000..73302cc --- /dev/null +++ b/spec/spec_helper.rb @@ -0,0 +1,29 @@ +$LOAD_PATH.unshift(File.dirname(__FILE__)) +$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), "..", "lib")) + +require "bundler" +Bundler.setup +require "typhoeus" +require "rspec" + +Dir[File.join(File.dirname(__FILE__), "support/**/*.rb")].each { |f| require f } + +RSpec.configure do |config| + config.order = :rand + + config.before(:suite) do + LocalhostServer.new(TESTSERVER.new, 3001) + end + + config.after do + Typhoeus::Pool.clear + Typhoeus::Expectation.clear + Typhoeus.before.clear + Typhoeus.on_complete.clear + Typhoeus.on_success.clear + Typhoeus.on_failure.clear + Typhoeus::Config.verbose = false + Typhoeus::Config.block_connection = false + Typhoeus::Config.memoize = false + end +end diff --git a/spec/support/localhost_server.rb b/spec/support/localhost_server.rb new file mode 100644 index 0000000..c5a7508 --- /dev/null +++ b/spec/support/localhost_server.rb @@ -0,0 +1,94 @@ +require 'rack' +require 'rack/handler/webrick' +require 'net/http' + +# The code for this is inspired by Capybara's server: +# http://github.com/jnicklas/capybara/blob/0.3.9/lib/capybara/server.rb +class LocalhostServer + READY_MESSAGE = "Server ready" + + class Identify + def initialize(app) + @app = app + end + + def call(env) + if env["PATH_INFO"] == "/__identify__" + [200, {}, [LocalhostServer::READY_MESSAGE]] + else + @app.call(env) + end + end + end + + attr_reader :port + + def initialize(rack_app, port = nil) + @port = port || find_available_port + @rack_app = rack_app + concurrently { boot } + wait_until(10, "Boot failed.") { booted? } + end + + private + + def find_available_port + server = TCPServer.new('127.0.0.1', 0) + server.addr[1] + ensure + server.close if server + end + + def boot + # Use WEBrick since it's part of the ruby standard library and is available on all ruby interpreters. + options = { :Port => port } + options.merge!(:AccessLog => [], :Logger => WEBrick::BasicLog.new(StringIO.new)) unless ENV['VERBOSE_SERVER'] + Rack::Handler::WEBrick.run(Identify.new(@rack_app), options) + end + + def booted? + res = ::Net::HTTP.get_response("localhost", '/__identify__', port) + if res.is_a?(::Net::HTTPSuccess) or res.is_a?(::Net::HTTPRedirection) + return res.body == READY_MESSAGE + end + rescue Errno::ECONNREFUSED, Errno::EBADF + return false + end + + def concurrently + if should_use_subprocess? + pid = Process.fork do + trap(:INT) { ::Rack::Handler::WEBrick.shutdown } + yield + exit # manually exit; otherwise this sub-process will re-run the specs that haven't run yet. + end + + at_exit do + Process.kill('INT', pid) + begin + Process.wait(pid) + rescue Errno::ECHILD + # ignore this error...I think it means the child process has already exited. + end + end + else + Thread.new { yield } + end + end + + def should_use_subprocess? + # !ENV['THREADED'] + false + end + + def wait_until(timeout, error_message, &block) + start_time = Time.now + + while true + return if yield + raise TimeoutError.new(error_message) if (Time.now - start_time) > timeout + sleep(0.05) + end + end +end + diff --git a/spec/support/memory_cache.rb b/spec/support/memory_cache.rb new file mode 100644 index 0000000..15cc56b --- /dev/null +++ b/spec/support/memory_cache.rb @@ -0,0 +1,15 @@ +class MemoryCache + attr_reader :memory + + def initialize + @memory = {} + end + + def get(request) + memory[request] + end + + def set(request, response) + memory[request] = response + end +end diff --git a/spec/support/server.rb b/spec/support/server.rb new file mode 100644 index 0000000..b7f6f06 --- /dev/null +++ b/spec/support/server.rb @@ -0,0 +1,116 @@ +#!/usr/bin/env ruby +require 'json' +require 'zlib' +require 'sinatra/base' +require 'rack/typhoeus' + +TESTSERVER = Sinatra.new do + set :logging, false + use Rack::Typhoeus::Middleware::ParamsDecoder + + fail_count = 0 + + post '/file' do + { + 'content-type' => params[:file][:type], + 'filename' => params[:file][:filename], + 'content' => params[:file][:tempfile].read, + 'request-content-type' => request.env['CONTENT_TYPE'] + }.to_json + end + + get '/multiple-headers' do + [200, { 'Set-Cookie' => %w[ foo bar ], 'Content-Type' => 'text/plain' }, ['']] + end + + get '/cookies-test' do + [200, { 'Set-Cookie' => %w(foo=bar bar=foo), 'Content-Type' => 'text/plain' }, ['']] + end + + get '/cookies-test2' do + [200, { 'Set-Cookie' => %w(foo2=bar bar2=foo), 'Content-Type' => 'text/plain' }, ['']] + end + + get '/fail/:number' do + if fail_count >= params[:number].to_i + "ok" + else + fail_count += 1 + error 500, "oh noes!" + end + end + + get '/fail_forever' do + error 500, "oh noes!" + end + + get '/redirect' do + redirect '/' + end + + get '/bad_redirect' do + redirect '/bad_redirect' + end + + get '/auth_basic/:username/:password' do + @auth ||= Rack::Auth::Basic::Request.new(request.env) + # Check that we've got a basic auth, and that it's credentials match the ones + # provided in the request + if @auth.provided? && @auth.basic? && @auth.credentials == [ params[:username], params[:password] ] + # auth is valid - confirm it + true + else + # invalid auth - request the authentication + response['WWW-Authenticate'] = %(Basic realm="Testing HTTP Auth") + throw(:halt, [401, "Not authorized\n"]) + end + end + + get '/auth_ntlm' do + # we're just checking for the existence if NTLM auth header here. It's validation + # is too troublesome and really doesn't bother is much, it's up to libcurl to make + # it valid + response['WWW-Authenticate'] = 'NTLM' + is_ntlm_auth = /^NTLM/ =~ request.env['HTTP_AUTHORIZATION'] + true if is_ntlm_auth + throw(:halt, [401, "Not authorized\n"]) if !is_ntlm_auth + end + + get '/gzipped' do + req_env = request.env.to_json + z = Zlib::Deflate.new + gzipped_env = z.deflate(req_env, Zlib::FINISH) + z.close + response['Content-Encoding'] = 'gzip' + gzipped_env + end + + get '/**' do + sleep params["delay"].to_i if params.has_key?("delay") + request.env.merge!(:body => request.body.read).to_json + end + + head '/**' do + sleep params["delay"].to_i if params.has_key?("delay") + end + + put '/**' do + request.env.merge!(:body => request.body.read).to_json + end + + post '/**' do + request.env.merge!(:body => request.body.read).to_json + end + + delete '/**' do + request.env.merge!(:body => request.body.read).to_json + end + + patch '/**' do + request.env.merge!(:body => request.body.read).to_json + end + + options '/**' do + request.env.merge!(:body => request.body.read).to_json + end +end diff --git a/spec/typhoeus/adapters/faraday_spec.rb b/spec/typhoeus/adapters/faraday_spec.rb new file mode 100644 index 0000000..716fb76 --- /dev/null +++ b/spec/typhoeus/adapters/faraday_spec.rb @@ -0,0 +1,339 @@ +if Gem::Version.new(RUBY_VERSION) >= Gem::Version.new("1.9.0") + require 'spec_helper' + require 'typhoeus/adapters/faraday' + + describe Faraday::Adapter::Typhoeus do + let(:base_url) { "http://localhost:3001" } + let(:adapter) { described_class.new(nil) } + let(:request) { Typhoeus::Request.new(base_url) } + let(:conn) do + Faraday.new(:url => base_url) do |faraday| + faraday.adapter :typhoeus + end + end + let(:response) { conn.get("/") } + + context "when parallel" do + it "returns a faraday response" do + response = nil + conn.in_parallel { response = conn.get("/") } + expect(response).to be_a(Faraday::Response) + end + + it "succeeds" do + response = nil + conn.in_parallel { response = conn.get("/") } + expect(response.status).to be(200) + end + end + + context "when not parallel" do + it "returns a faraday response" do + expect(response).to be_a(Faraday::Response) + end + + it "succeeds" do + expect(response.status).to be(200) + end + end + + context "when a response is stubbed" do + before do + stub = Typhoeus::Response.new \ + :code => 200, + :headers => { "Foo" => "2", "Bar" => "3" }, + :body => "Hello", + :mock => true + + Typhoeus.stub(base_url + '/').and_return(stub) + end + + it 'stubs the status code' do + expect(response.status).to eq(200) + end + + it 'stubs the response body' do + expect(response.body).to eq("Hello") + end + + it 'stubs the headers' do + expect(response.headers).to eq("Foo" => "2", "Bar" => "3") + end + end + + describe "#initialize" do + let(:request) { adapter.method(:typhoeus_request).call({}) } + + context "when typhoeus request options specified" do + let(:adapter) { described_class.new(nil, { :forbid_reuse => true, :maxredirs => 1 }) } + + it "should set option for request" do + expect(request.options[:forbid_reuse]).to be_truthy + expect(request.options[:maxredirs]).to eq(1) + end + end + end + + describe "#perform_request" do + let(:env) { {} } + + context "when body" do + let(:env) { { :body => double(:read => "body") } } + + it "reads body" do + expect(adapter.method(:read_body).call(env)).to eq("body") + end + end + + context "parallel_manager" do + context "when given" do + let(:env) { { :parallel_manager => double(:queue => true), :ssl => {}, :request => {} } } + + it "uses" do + adapter.method(:perform_request).call(env) + end + end + + context "when not given" do + let(:env) { { :method => :get, :ssl => {}, :request => {} } } + + it "falls back to single" do + expect(Typhoeus::Request).to receive(:new).and_return(double(:options => {}, :on_complete => [], :run => true)) + adapter.method(:perform_request).call(env) + end + end + end + end + + describe "#request" do + let(:env) do + { :url => "url", :method => :get, :body => "body", :request_headers => {}, :ssl => {}, :request => {} } + end + + let(:request) { adapter.method(:request).call(env) } + + it "returns request" do + expect(request).to be_a(Typhoeus::Request) + end + + it "sets url" do + expect(request.base_url).to eq("url") + end + + it "sets http method" do + expect(request.original_options[:method]).to eq(:get) + end + + it "sets body" do + expect(request.original_options[:body]).to eq("body") + end + + it "sets headers" do + expect(request.original_options[:headers]).to eq({}) + end + + it "sets on_complete callback" do + expect(request.on_complete.size).to eq(1) + end + end + + context "when the connection failed" do + before do + stub = Typhoeus::Response.new \ + :response_code => 0, + :return_code => 0, + :mock => true + + Typhoeus.stub(base_url + '/').and_return(stub) + end + + context "when parallel" do + it "isn't successful" do + response = nil + conn.in_parallel { response = conn.get("/") } + expect(response.success?).to be_falsey + end + + it "translates the response code into an error message" do + response = nil + conn.in_parallel { response = conn.get("/") } + expect(response.env[:typhoeus_return_message]).to eq("No error") + end + end + + context "when not parallel" do + it "raises an error" do + expect { conn.get("/") }.to raise_error(Faraday::ConnectionFailed, "No error") + end + end + end + + describe "#configure_socket" do + let(:env) { { :request => { :bind => { :host => "interface" } } } } + + before { adapter.method(:configure_socket).call(request, env) } + + context "when host" do + it "sets interface" do + expect(request.options[:interface]).to eq("interface") + end + end + end + + describe "#configure_timeout" do + before { adapter.method(:configure_timeout).call(request, env) } + + context "when timeout" do + let(:env) { { :request => { :timeout => 1 } } } + + it "sets timeout_ms" do + expect(request.options[:timeout_ms]).to eq(1000) + end + end + + context "when open_timeout" do + let(:env) { { :request => { :open_timeout => 1 } } } + + it "sets connecttimeout_ms" do + expect(request.options[:connecttimeout_ms]).to eq(1000) + end + end + end + + describe "#configure_proxy" do + before { adapter.method(:configure_proxy).call(request, env) } + + context "when proxy" do + let(:env) { { :request => { :proxy => { :uri => double(:scheme => 'http', :host => "localhost", :port => "3001") } } } } + + it "sets proxy" do + expect(request.options[:proxy]).to eq("http://localhost:3001") + end + + context "when username and password" do + let(:env) do + { :request => { :proxy => { + :uri => double(:scheme => 'http', :host => :a, :port => :b), + :user => "a", + :password => "b" + } } } + end + + it "sets proxyuserpwd" do + expect(request.options[:proxyuserpwd]).to eq("a:b") + end + end + end + end + + describe "#configure_ssl" do + before { adapter.method(:configure_ssl).call(request, env) } + + context "when version" do + let(:env) { { :ssl => { :version => "a" } } } + + it "sets sslversion" do + expect(request.options[:sslversion]).to eq("a") + end + end + + context "when client_cert" do + let(:env) { { :ssl => { :client_cert => "a" } } } + + it "sets sslcert" do + expect(request.options[:sslcert]).to eq("a") + end + end + + context "when client_key" do + let(:env) { { :ssl => { :client_key => "a" } } } + + it "sets sslkey" do + expect(request.options[:sslkey]).to eq("a") + end + end + + context "when ca_file" do + let(:env) { { :ssl => { :ca_file => "a" } } } + + it "sets cainfo" do + expect(request.options[:cainfo]).to eq("a") + end + end + + context "when ca_path" do + let(:env) { { :ssl => { :ca_path => "a" } } } + + it "sets capath" do + expect(request.options[:capath]).to eq("a") + end + end + + context "when client_cert_passwd" do + let(:env) { { :ssl => { :client_cert_passwd => "a" } } } + + it "sets keypasswd to the value of client_cert_passwd" do + expect(request.options[:keypasswd]).to eq("a") + end + end + + context "when client_certificate_password" do + let(:env) { { :ssl => { :client_certificate_password => "a" } } } + + it "sets keypasswd to the value of client_cert_passwd" do + expect(request.options[:keypasswd]).to eq("a") + end + end + + context "when no client_cert_passwd" do + let(:env) { { :ssl => { } } } + + it "does not set keypasswd on options" do + expect(request.options).not_to have_key :keypasswd + end + end + + context "when verify is false" do + let(:env) { { :ssl => { :verify => false } } } + + it "sets ssl_verifyhost to 0" do + expect(request.options[:ssl_verifyhost]).to eq(0) + end + + it "sets ssl_verifypeer to false" do + expect(request.options[:ssl_verifypeer]).to be_falsey + end + end + + context "when verify is true" do + let(:env) { { :ssl => { :verify => true } } } + + it "sets ssl_verifyhost to 2" do + expect(request.options[:ssl_verifyhost]).to eq(2) + end + + it "sets ssl_verifypeer to true" do + expect(request.options[:ssl_verifypeer]).to be_truthy + end + end + end + + describe "#parallel?" do + context "when parallel_manager" do + let(:env) { { :parallel_manager => true } } + + it "returns true" do + expect(adapter.method(:parallel?).call(env)).to be_truthy + end + end + + context "when no parallel_manager" do + let(:env) { { :parallel_manager => nil } } + + it "returns false" do + expect(adapter.method(:parallel?).call(env)).to be_falsey + end + end + end + end +end diff --git a/spec/typhoeus/cache/dalli_spec.rb b/spec/typhoeus/cache/dalli_spec.rb new file mode 100644 index 0000000..1ea2ce0 --- /dev/null +++ b/spec/typhoeus/cache/dalli_spec.rb @@ -0,0 +1,41 @@ +if Gem::Version.new(RUBY_VERSION) >= Gem::Version.new("1.9.0") + require 'dalli' + require 'typhoeus/cache/dalli' + require 'spec_helper' + + describe Typhoeus::Cache::Dalli do + let(:dalli) { instance_double(Dalli::Client) } + let(:cache) { Typhoeus::Cache::Dalli.new(dalli) } + + let(:base_url) { "localhost:3001" } + let(:request) { Typhoeus::Request.new(base_url, {:method => :get}) } + let(:response) { Typhoeus::Response.new(:response_code => 0, :return_code => 0, :mock => true) } + + describe "#set" do + it "sends the request to Dalli" do + expect(dalli).to receive(:set).with(request.cache_key, response, nil) + + cache.set(request, response) + end + end + + describe "#get" do + it "returns nil when the key is not in the cache" do + expect(dalli).to receive(:get).with(request.cache_key).and_return(nil) + + expect(cache.get(request)).to be_nil + end + + it "returns the cached response when the key is in cache" do + expect(dalli).to receive(:get).with(request.cache_key).and_return(response) + + result = cache.get(request) + expect(result).to_not be_nil + expect(result.response_code).to eq(response.response_code) + expect(result.return_code).to eq(response.return_code) + expect(result.headers).to eq(response.headers) + expect(result.body).to eq(response.body) + end + end + end +end diff --git a/spec/typhoeus/cache/redis_spec.rb b/spec/typhoeus/cache/redis_spec.rb new file mode 100644 index 0000000..9ee5941 --- /dev/null +++ b/spec/typhoeus/cache/redis_spec.rb @@ -0,0 +1,41 @@ +require 'redis' +require 'typhoeus/cache/redis' +require 'spec_helper' + +describe Typhoeus::Cache::Redis do + let(:redis) { instance_double(Redis) } + let(:cache) { Typhoeus::Cache::Redis.new(redis) } + + let(:base_url) { "localhost:3001" } + let(:request) { Typhoeus::Request.new(base_url, {:method => :get}) } + let(:response) { Typhoeus::Response.new(:response_code => 0, :return_code => 0, :mock => true) } + let(:serialized_response) { Marshal.dump(response) } + + describe "#set" do + it "sends the serialized request to Redis" do + expect(redis).to receive(:set).with(request.cache_key, serialized_response) + expect(redis).to_not receive(:expire).with(request.cache_key, request.cache_ttl) + + cache.set(request, response) + end + end + + describe "#get" do + it "returns nil when the key is not in Redis" do + expect(redis).to receive(:get).with(request.cache_key).and_return(nil) + + expect(cache.get(request)).to be_nil + end + + it "returns the cached response when the key is in Redis" do + expect(redis).to receive(:get).with(request.cache_key).and_return(serialized_response) + + result = cache.get(request) + expect(result).to_not be_nil + expect(result.response_code).to eq(response.response_code) + expect(result.return_code).to eq(response.return_code) + expect(result.headers).to eq(response.headers) + expect(result.body).to eq(response.body) + end + end +end diff --git a/spec/typhoeus/config_spec.rb b/spec/typhoeus/config_spec.rb new file mode 100644 index 0000000..ac721c5 --- /dev/null +++ b/spec/typhoeus/config_spec.rb @@ -0,0 +1,15 @@ +require 'spec_helper' + +describe Typhoeus::Config do + let(:config) { Typhoeus::Config } + + [:block_connection, :memoize, :verbose, :cache, :user_agent, :proxy].each do |name| + it "responds to #{name}" do + expect(config).to respond_to(name) + end + + it "responds to #{name}=" do + expect(config).to respond_to("#{name}=") + end + end +end diff --git a/spec/typhoeus/easy_factory_spec.rb b/spec/typhoeus/easy_factory_spec.rb new file mode 100644 index 0000000..cc1e9bc --- /dev/null +++ b/spec/typhoeus/easy_factory_spec.rb @@ -0,0 +1,143 @@ +require 'spec_helper' + +describe Typhoeus::EasyFactory do + let(:base_url) { "http://localhost:3001" } + let(:hydra) { Typhoeus::Hydra.new(:max_concurrency => 1) } + let(:options) { {} } + let(:request) { Typhoeus::Request.new(base_url, options) } + let(:easy_factory) { described_class.new(request, hydra) } + + describe "#get" do + context "when option[:cache_ttl]" do + let(:options) { {:cache_ttl => 1} } + + it "creates Ethon::Easy" do + expect(easy_factory.get).to be_a(Ethon::Easy) + end + end + + context "timeouts" do + it "sets nosignal to true by default" do + expect(easy_factory.easy).to receive(:http_request).with(anything(), anything(), hash_including(:nosignal => true)) + easy_factory.get + end + + context "when timeout is not a whole number and timeout_ms is not set" do + let(:options) { {:timeout => 0.1} } + it "ceils timeout and sets timeout_ms" do + expect(easy_factory.easy).to receive(:http_request).with(anything(), anything(), hash_including(:timeout_ms => 100, :timeout => 1)) + easy_factory.get + end + end + + context "when timeout is not a whole number and timeout_ms is set" do + let(:options) { {:timeout => 0.1, :timeout_ms => 123} } + it "ceils timeout and does not change timeout_ms" do + expect(easy_factory.easy).to receive(:http_request).with(anything(), anything(), hash_including(:timeout_ms => 123, :timeout => 1)) + easy_factory.get + end + end + + context "when connecttimeout is not a whole number and connecttimeout_ms is not set" do + let(:options) { {:connecttimeout => 0.1} } + it "ceils connecttimeout and sets connecttimeout_ms" do + expect(easy_factory.easy).to receive(:http_request).with(anything(), anything(), hash_including(:connecttimeout_ms => 100, :connecttimeout => 1)) + easy_factory.get + end + end + + context "when connecttimeout is not a whole number and connecttimeout_ms is set" do + let(:options) { {:connecttimeout => 0.1, :connecttimeout_ms => 123} } + it "ceils connecttimeout and does not change connecttimeout_ms" do + expect(easy_factory.easy).to receive(:http_request).with(anything(), anything(), hash_including(:connecttimeout_ms => 123, :connecttimeout => 1)) + easy_factory.get + end + end + + + end + + context "when invalid option" do + let(:options) { {:invalid => 1} } + + it "reraises" do + expect{ easy_factory.get }.to raise_error(Ethon::Errors::InvalidOption) + end + end + + context "when removed option" do + let(:options) { {:cache_timeout => 1} } + + it "reraises with help" do + expect{ easy_factory.get }.to raise_error( + Ethon::Errors::InvalidOption, /The option cache_timeout was removed/ + ) + end + end + + context "when changed option" do + let(:options) { {:proxy_auth_method => 1} } + + it "reraises with help" do + expect{ easy_factory.get }.to raise_error( + Ethon::Errors::InvalidOption, /Please try proxyauth instead of proxy_auth_method/ + ) + end + end + + context "when renamed option" do + let(:options) { {:connect_timeout => 1} } + + it "warns" do + expect(easy_factory).to receive(:warn).with( + "Deprecated option connect_timeout. Please use connecttimeout instead." + ) + easy_factory.get + end + + it "passes correct option" do + expect(easy_factory).to receive(:warn) + expect(easy_factory.easy).to receive(:connecttimeout=).with(1) + easy_factory.get + end + end + end + + describe "#set_callback" do + it "sets easy.on_progress callback when an on_progress callback is provided" do + request.on_progress { 1 } + expect(easy_factory.easy).to receive(:on_progress) + easy_factory.send(:set_callback) + end + + it "sets easy.on_complete callback" do + expect(easy_factory.easy).to receive(:on_complete) + easy_factory.send(:set_callback) + end + + it "finishes request" do + easy_factory.send(:set_callback) + expect(request).to receive(:finish) + easy_factory.easy.complete + end + + it "resets easy" do + easy_factory.send(:set_callback) + expect(easy_factory.easy).to receive(:reset) + easy_factory.easy.complete + end + + it "pushes easy back into the pool" do + easy_factory.send(:set_callback) + easy_factory.easy.complete + expect(Typhoeus::Pool.send(:easies)).to include(easy_factory.easy) + end + + it "adds next request" do + easy_factory.hydra.instance_variable_set(:@queued_requests, [request]) + expect(easy_factory.hydra).to receive(:add).with(request) + easy_factory.send(:set_callback) + easy_factory.easy.complete + end + end +end diff --git a/spec/typhoeus/errors/no_stub_spec.rb b/spec/typhoeus/errors/no_stub_spec.rb new file mode 100644 index 0000000..5815cea --- /dev/null +++ b/spec/typhoeus/errors/no_stub_spec.rb @@ -0,0 +1,13 @@ +require 'spec_helper' + +describe Typhoeus::Errors::NoStub do + let(:base_url) { "localhost:3001" } + let(:request) { Typhoeus::Request.new(base_url) } + let(:message) { "The connection is blocked and no stub defined: " } + + subject { Typhoeus::Errors::NoStub } + + it "displays the request url" do + expect { raise subject.new(request) }.to raise_error(subject, message + base_url) + end +end diff --git a/spec/typhoeus/expectation_spec.rb b/spec/typhoeus/expectation_spec.rb new file mode 100644 index 0000000..9dfb2b0 --- /dev/null +++ b/spec/typhoeus/expectation_spec.rb @@ -0,0 +1,280 @@ +require 'spec_helper' + +describe Typhoeus::Expectation do + let(:options) { {} } + let(:base_url) { "www.example.com" } + let(:expectation) { described_class.new(base_url, options) } + + describe ".new" do + it "sets base_url" do + expect(expectation.instance_variable_get(:@base_url)).to eq(base_url) + end + + it "sets options" do + expect(expectation.instance_variable_get(:@options)).to eq(options) + end + + it "initializes response_counter" do + expect(expectation.instance_variable_get(:@response_counter)).to eq(0) + end + end + + describe ".all" do + context "when @expectations nil" do + it "returns empty array" do + expect(Typhoeus::Expectation.all).to eq([]) + end + end + + context "when @expectations not nil" do + let(:expectations) { [1] } + + it "returns @expectations" do + Typhoeus::Expectation.instance_variable_set(:@expectations, expectations) + expect(Typhoeus::Expectation.all).to be(expectations) + end + end + end + + describe ".clear" do + let(:expectations) { double(:clear) } + + it "clears all" do + expect(expectations).to receive(:clear) + Typhoeus::Expectation.instance_variable_set(:@expectations, expectations) + Typhoeus::Expectation.clear + Typhoeus::Expectation.instance_variable_set(:@expectations, nil) + end + end + + describe ".response_for" do + let(:request) { Typhoeus::Request.new("") } + let(:stubbed_response) { Typhoeus::Response.new } + + it "finds a matching expectation and returns its next response" do + Typhoeus::Expectation.all << expectation + expect(expectation).to receive(:matches?).with(request).and_return(true) + expect(expectation).to receive(:response).with(request).and_return(stubbed_response) + + response = Typhoeus::Expectation.response_for(request) + + expect(response).to be(stubbed_response) + end + + it "returns nil if no matching expectation is found" do + response = Typhoeus::Expectation.response_for(request) + expect(response).to be(nil) + end + end + + describe "#stubbed_from" do + it "sets value" do + expectation.stubbed_from(:webmock) + expect(expectation.from).to eq(:webmock) + end + + it "returns self" do + expect(expectation.stubbed_from(:webmock)).to be(expectation) + end + end + + describe "#and_return" do + context "when value" do + it "adds to responses" do + expectation.and_return(1) + expect(expectation.responses).to eq([1]) + end + end + + context "when array" do + it "adds to responses" do + expectation.and_return([1, 2]) + expect(expectation.responses).to eq([1, 2]) + end + end + + context "when block" do + it "adds to responses" do + block = Proc.new {} + expectation.and_return(&block) + expect(expectation.responses).to eq([block]) + end + end + end + + describe "#responses" do + it "returns responses" do + expect(expectation.responses).to be_a(Array) + end + end + + describe "#response" do + let(:request) { Typhoeus::Request.new("") } + + before { expectation.instance_variable_set(:@responses, responses) } + + context "when one response" do + context "is pre-constructed" do + let(:responses) { [Typhoeus::Response.new] } + + it "returns response" do + expect(expectation.response(request)).to be(responses[0]) + end + end + + context "is lazily-constructed" do + def construct_response(request) + @request_from_response_construction = request + lazily_constructed_response + end + + let(:lazily_constructed_response) { Typhoeus::Response.new } + let(:responses) { [ Proc.new { |request| construct_response(request) } ] } + + it "returns response" do + expect(expectation.response(request)).to be(lazily_constructed_response) + expect(@request_from_response_construction).to be(request) + end + end + end + + context "when multiple responses" do + let(:responses) { [Typhoeus::Response.new, Typhoeus::Response.new, Typhoeus::Response.new] } + + it "returns one by one" do + 3.times do |i| + expect(expectation.response(request)).to be(responses[i]) + end + end + end + end + + describe "#matches?" do + let(:request) { double(:base_url => nil) } + + it "calls url_match?" do + expect(expectation).to receive(:url_match?) + expectation.matches?(request) + end + + it "calls options_match?" do + expect(expectation).to receive(:url_match?).and_return(true) + expect(expectation).to receive(:options_match?) + expectation.matches?(request) + end + end + + describe "#url_match?" do + let(:request_url) { "www.example.com" } + let(:request) { Typhoeus::Request.new(request_url) } + let(:url_match) { expectation.method(:url_match?).call(request.base_url) } + + context "when string" do + context "when match" do + it "returns true" do + expect(url_match).to be_truthy + end + end + + context "when no match" do + let(:base_url) { "no_match" } + + it "returns false" do + expect(url_match).to be_falsey + end + end + end + + context "when regexp" do + context "when match" do + let(:base_url) { /example/ } + + it "returns true" do + expect(url_match).to be_truthy + end + end + + context "when no match" do + let(:base_url) { /nomatch/ } + + it "returns false" do + expect(url_match).to be_falsey + end + + context "with nil request_url" do + let(:request_url) { nil } + + it "returns false" do + expect(url_match).to be_falsey + end + end + end + end + + context "when nil" do + let(:base_url) { nil } + + it "returns true" do + expect(url_match).to be_truthy + end + end + + context "when not string, regexp, nil" do + let(:base_url) { 1 } + + it "returns false" do + expect(url_match).to be_falsey + end + end + end + + describe "options_match?" do + let(:request_options) { {} } + let(:request) { Typhoeus::Request.new(nil, request_options) } + let(:options_match) { expectation.method(:options_match?).call(request) } + + context "when match" do + let(:options) { { :a => 1 } } + let(:request_options) { options } + + it "returns true" do + expect(options_match).to be_truthy + end + end + + context "when options are a subset from request_options" do + let(:options) { { :a => 1 } } + let(:request_options) { { :a => 1, :b => 2 } } + + it "returns true" do + expect(options_match).to be_truthy + end + end + + context "when options are nested" do + let(:options) { { :a => { :b => 1 } } } + let(:request_options) { options } + + it "returns true" do + expect(options_match).to be_truthy + end + end + + context "when options contains an array" do + let(:options) { { :a => [1, 2] } } + let(:request_options) { options } + + it "returns true" do + expect(options_match).to be_truthy + end + end + + context "when no match" do + let(:options) { { :a => 1 } } + + it "returns false" do + expect(options_match).to be_falsey + end + end + end +end diff --git a/spec/typhoeus/hydra/addable_spec.rb b/spec/typhoeus/hydra/addable_spec.rb new file mode 100644 index 0000000..4cb8e1a --- /dev/null +++ b/spec/typhoeus/hydra/addable_spec.rb @@ -0,0 +1,22 @@ +require 'spec_helper' + +describe Typhoeus::Hydra::Addable do + let(:hydra) { Typhoeus::Hydra.new() } + let(:request) { Typhoeus::Request.new("localhost:3001", {:method => :get}) } + + it "asks easy factory for an easy" do + multi = double + expect(Typhoeus::EasyFactory).to receive(:new).with(request, hydra).and_return(double(:get => 1)) + expect(hydra).to receive(:multi).and_return(multi) + expect(multi).to receive(:add).with(1) + hydra.add(request) + end + + it "adds easy to multi" do + multi = double + expect(Typhoeus::EasyFactory).to receive(:new).with(request, hydra).and_return(double(:get => 1)) + expect(hydra).to receive(:multi).and_return(multi) + expect(multi).to receive(:add).with(1) + hydra.add(request) + end +end diff --git a/spec/typhoeus/hydra/before_spec.rb b/spec/typhoeus/hydra/before_spec.rb new file mode 100644 index 0000000..e17e043 --- /dev/null +++ b/spec/typhoeus/hydra/before_spec.rb @@ -0,0 +1,98 @@ +require 'spec_helper' + +describe Typhoeus::Hydra::Before do + let(:request) { Typhoeus::Request.new("") } + let(:hydra) { Typhoeus::Hydra.new } + let(:receive_counter) { double :mark => :twain } + + describe "#add" do + context "when before" do + context "when one" do + it "executes" do + Typhoeus.before { |r| receive_counter.mark } + expect(receive_counter).to receive(:mark) + hydra.add(request) + end + + context "when true" do + it "calls super" do + Typhoeus.before { true } + expect(Typhoeus::Expectation).to receive(:response_for) + hydra.add(request) + end + end + + context "when falsy" do + context "when queue requests" do + let(:queued_request) { Typhoeus::Request.new("") } + + before { hydra.queue(queued_request) } + + it "dequeues" do + Typhoeus.before { false } + hydra.add(request) + expect(hydra.queued_requests).to be_empty + end + end + + context "when false" do + it "doesn't call super" do + Typhoeus.before { false } + expect(Typhoeus::Expectation).to receive(:response_for).never + hydra.add(request) + end + end + + context "when response" do + it "doesn't call super" do + Typhoeus.before { Typhoeus::Response.new } + expect(Typhoeus::Expectation).to receive(:response_for).never + hydra.add(request) + end + end + end + end + + context "when multi" do + context "when all true" do + before { 3.times { Typhoeus.before { |r| receive_counter.mark } } } + + it "calls super" do + expect(Typhoeus::Expectation).to receive(:response_for) + hydra.add(request) + end + + it "executes all" do + expect(receive_counter).to receive(:mark).exactly(3).times + hydra.add(request) + end + end + + context "when middle false" do + before do + Typhoeus.before { |r| receive_counter.mark } + Typhoeus.before { |r| receive_counter.mark; nil } + Typhoeus.before { |r| receive_counter.mark } + end + + it "doesn't call super" do + expect(Typhoeus::Expectation).to receive(:response_for).never + hydra.add(request) + end + + it "executes only two" do + expect(receive_counter).to receive(:mark).exactly(2).times + hydra.add(request) + end + end + end + end + + context "when no before" do + it "calls super" do + expect(Typhoeus::Expectation).to receive(:response_for) + hydra.add(request) + end + end + end +end diff --git a/spec/typhoeus/hydra/block_connection_spec.rb b/spec/typhoeus/hydra/block_connection_spec.rb new file mode 100644 index 0000000..df3e945 --- /dev/null +++ b/spec/typhoeus/hydra/block_connection_spec.rb @@ -0,0 +1,18 @@ +require 'spec_helper' + +describe Typhoeus::Hydra::BlockConnection do + let(:base_url) { "localhost:3001" } + let(:hydra) { Typhoeus::Hydra.new() } + let(:request) { Typhoeus::Request.new(base_url, {:method => :get}) } + + describe "add" do + context "when block_connection activated" do + before { Typhoeus::Config.block_connection = true } + after { Typhoeus::Config.block_connection = false } + + it "raises" do + expect{ hydra.add(request) }.to raise_error(Typhoeus::Errors::NoStub) + end + end + end +end diff --git a/spec/typhoeus/hydra/cacheable_spec.rb b/spec/typhoeus/hydra/cacheable_spec.rb new file mode 100644 index 0000000..6e2f0ab --- /dev/null +++ b/spec/typhoeus/hydra/cacheable_spec.rb @@ -0,0 +1,88 @@ +require 'spec_helper' + +describe Typhoeus::Hydra::Cacheable do + let(:base_url) { "localhost:3001" } + let(:hydra) { Typhoeus::Hydra.new() } + let(:request) { Typhoeus::Request.new(base_url, {:method => :get}) } + let(:response) { Typhoeus::Response.new } + let(:cache) { MemoryCache.new } + + describe "add" do + context "when cache activated" do + before { Typhoeus::Config.cache = cache } + after { Typhoeus::Config.cache = false } + + context "when request new" do + it "sets no response" do + hydra.add(request) + expect(request.response).to be_nil + end + + it "doesn't call complete" do + expect(request).to receive(:complete).never + hydra.add(request) + end + end + + context "when request in memory" do + before { cache.memory[request] = response } + + it "returns response with cached status" do + hydra.add(request) + expect(response.cached?).to be_truthy + end + + context "when no queued requests" do + it "finishes request" do + expect(request).to receive(:finish).with(response) + hydra.add(request) + expect(response.cached?).to be_truthy + end + end + + context "when queued requests" do + let(:queued_request) { Typhoeus::Request.new(base_url, {:method => :get}) } + + before { cache.memory[queued_request] = response } + + it "finishes both requests" do + hydra.queue(queued_request) + expect(request).to receive(:finish).with(response) + expect(queued_request).to receive(:finish).with(response) + hydra.add(request) + end + end + end + + context "when cache is specified on a request" do + before { Typhoeus::Config.cache = false } + + context "when cache is false" do + let(:non_cached_request) { Typhoeus::Request.new(base_url, {:method => :get, :cache => false}) } + + it "initiates an HTTP call" do + expect(Typhoeus::EasyFactory).to receive(:new).with(non_cached_request, hydra).and_call_original + + hydra.add(non_cached_request) + end + end + + context "when cache is defined" do + let(:cached_request) { Typhoeus::Request.new(base_url, {:method => :get, :cache => cache}) } + + before { cache.memory[cached_request] = response } + + it "uses the cache instead of making a new request" do + expect(Typhoeus::EasyFactory).not_to receive(:new) + + hydra.add(cached_request) + + expect(cached_request.response).to be_cached + expect(cached_request.response).to eq(response) + end + end + end + + end + end +end diff --git a/spec/typhoeus/hydra/memoizable_spec.rb b/spec/typhoeus/hydra/memoizable_spec.rb new file mode 100644 index 0000000..c04f299 --- /dev/null +++ b/spec/typhoeus/hydra/memoizable_spec.rb @@ -0,0 +1,53 @@ +require 'spec_helper' + +describe Typhoeus::Hydra::Memoizable do + let(:base_url) { "localhost:3001" } + let(:hydra) { Typhoeus::Hydra.new() } + let(:request) { Typhoeus::Request.new(base_url) } + + describe "add" do + context "when memoization activated" do + before { Typhoeus::Config.memoize = true } + + context "when request new" do + it "sets no response" do + hydra.add(request) + expect(request.response).to be_nil + end + + it "doesn't call complete" do + expect(request).to receive(:complete).never + hydra.add(request) + end + end + + context "when request in memory" do + let(:response) { Typhoeus::Response.new } + before { hydra.memory[request] = response } + + it "finishes request" do + expect(request).to receive(:finish).with(response, true) + hydra.add(request) + end + + context "when queued request" do + let(:queued_request) { Typhoeus::Request.new(base_url) } + + it "dequeues" do + hydra.queue(queued_request) + expect(request).to receive(:finish).with(response, true) + expect(queued_request).to receive(:finish).with(response, true) + hydra.add(request) + end + end + end + end + end + + describe "#run" do + it "clears memory before starting" do + expect(hydra.memory).to receive(:clear) + hydra.run + end + end +end diff --git a/spec/typhoeus/hydra/queueable_spec.rb b/spec/typhoeus/hydra/queueable_spec.rb new file mode 100644 index 0000000..d9fc87a --- /dev/null +++ b/spec/typhoeus/hydra/queueable_spec.rb @@ -0,0 +1,98 @@ +require 'spec_helper' + +describe Typhoeus::Hydra::Queueable do + let(:base_url) { "localhost:3001" } + let(:options) { {} } + let(:hydra) { Typhoeus::Hydra.new(options) } + + describe "#queue" do + let(:request) { Typhoeus::Request.new("") } + + it "accepts requests" do + hydra.queue(request) + end + + it "sets hydra on request" do + hydra.queue(request) + expect(request.hydra).to eq(hydra) + end + + it "adds to queued requests" do + hydra.queue(request) + expect(hydra.queued_requests).to include(request) + end + + it "adds to front of queued requests" do + hydra.queue_front(request) + expect(hydra.queued_requests.first).to be(request) + end + end + + describe "#abort" do + before { hydra.queued_requests << 1 } + + it "clears queue" do + hydra.abort + expect(hydra.queued_requests).to be_empty + end + end + + describe "#dequeue_many" do + before do + requests.each { |r| hydra.queue r } + end + + context "when no request queued" do + let(:requests) { [] } + + it "does nothing" do + expect(hydra).to_not receive(:add) + hydra.dequeue_many + end + end + + context "when request queued" do + let(:first) { Typhoeus::Request.new("localhost:3001/first") } + let(:requests) { [first] } + + it "adds request from queue to multi" do + expect(hydra).to receive(:add).with(first) + hydra.dequeue_many + end + end + + context "when three request queued" do + let(:first) { Typhoeus::Request.new("localhost:3001/first") } + let(:second) { Typhoeus::Request.new("localhost:3001/second") } + let(:third) { Typhoeus::Request.new("localhost:3001/third") } + let(:requests) { [first, second, third] } + + it "adds requests from queue to multi" do + expect(hydra).to receive(:add).with(first) + expect(hydra).to receive(:add).with(second) + expect(hydra).to receive(:add).with(third) + hydra.dequeue_many + end + + context "when max_concurrency is two" do + let(:options) { {:max_concurrency => 2} } + it "adds requests from queue to multi" do + expect(hydra).to receive(:add).with(first) + expect(hydra).to receive(:add).with(second) + expect(hydra).to_not receive(:add).with(third) + hydra.dequeue_many + end + end + + context "when max_concurrency is a string" do + let(:options) { {:max_concurrency => "2"} } + it "adds requests from queue to multi" do + expect(hydra).to receive(:add).with(first) + expect(hydra).to receive(:add).with(second) + expect(hydra).to_not receive(:add).with(third) + hydra.dequeue_many + end + end + end + end +end diff --git a/spec/typhoeus/hydra/runnable_spec.rb b/spec/typhoeus/hydra/runnable_spec.rb new file mode 100644 index 0000000..1e0237e --- /dev/null +++ b/spec/typhoeus/hydra/runnable_spec.rb @@ -0,0 +1,137 @@ +require 'spec_helper' + +describe Typhoeus::Hydra::Runnable do + let(:base_url) { "localhost:3001" } + let(:options) { {} } + let(:hydra) { Typhoeus::Hydra.new(options) } + let(:receive_counter) { double :mark => :twain } + + describe "#run" do + let(:requests) { [] } + + before do + requests.each { |r| hydra.queue r } + end + + it "runs multi#dequeue_many" do + expect(hydra).to receive(:dequeue_many) + hydra.run + end + + it "runs multi#perform" do + expect(hydra.multi).to receive(:perform) + hydra.run + end + + context "when request queued" do + let(:first) { Typhoeus::Request.new("localhost:3001/first") } + let(:requests) { [first] } + + it "sends" do + hydra.run + expect(first.response).to be + end + end + + context "when three request queued" do + let(:first) { Typhoeus::Request.new("localhost:3001/first") } + let(:second) { Typhoeus::Request.new("localhost:3001/second") } + let(:third) { Typhoeus::Request.new("localhost:3001/third") } + let(:requests) { [first, second, third] } + + it "sends first" do + hydra.run + expect(first.response).to be + end + + it "sends second" do + hydra.run + expect(second.response).to be + end + + it "sends third" do + hydra.run + expect(third.response).to be + end + + it "sends first first" do + first.on_complete do + expect(second.response).to be_nil + expect(third.response).to be_nil + end + end + + it "sends second second" do + first.on_complete do + expect(first.response).to be + expect(third.response).to be_nil + end + end + + it "sends thirds last" do + first.on_complete do + expect(second.response).to be + expect(third.response).to be + end + end + end + + context "when really queued request" do + let(:options) { {:max_concurrency => 1} } + let(:first) { Typhoeus::Request.new("localhost:3001/first") } + let(:second) { Typhoeus::Request.new("localhost:3001/second") } + let(:third) { Typhoeus::Request.new("localhost:3001/third") } + let(:requests) { [first, second, third] } + + it "sends first" do + hydra.run + expect(first.response).to be + end + + it "sends second" do + hydra.run + expect(second.response).to be + end + + it "sends third" do + hydra.run + expect(third.response).to be + end + end + + context "when request queued in callback" do + let(:first) do + Typhoeus::Request.new("localhost:3001/first").tap do |r| + r.on_complete{ hydra.queue(second) } + end + end + let(:second) { Typhoeus::Request.new("localhost:3001/second") } + let(:requests) { [first] } + + before { Typhoeus.on_complete { |r| receive_counter.mark } } + after { Typhoeus.on_complete.clear; Typhoeus.before.clear } + + context "when real request" do + context "when max_concurrency default" do + let(:options) { {} } + + it "calls on_complete callback once for every response" do + expect(receive_counter).to receive(:mark).exactly(2).times + hydra.run + end + end + end + + context "when no real request" do + context "when before hook returns and finishes response" do + before { Typhoeus.before{ |request| request.finish(Typhoeus::Response.new) } } + + it "simulates real multi run and adds and finishes both requests" do + expect(receive_counter).to receive(:mark).exactly(2).times + hydra.run + end + end + end + end + end +end diff --git a/spec/typhoeus/hydra/stubbable_spec.rb b/spec/typhoeus/hydra/stubbable_spec.rb new file mode 100644 index 0000000..6b64cc0 --- /dev/null +++ b/spec/typhoeus/hydra/stubbable_spec.rb @@ -0,0 +1,48 @@ +require 'spec_helper' + +describe Typhoeus::Hydra::Stubbable do + let(:base_url) { "localhost:3001" } + let(:request) { Typhoeus::Request.new(base_url) } + let(:response) { Typhoeus::Response.new } + let(:hydra) { Typhoeus::Hydra.new } + + before { Typhoeus.stub(base_url).and_return(response) } + + describe "#add" do + it "checks expectations" do + hydra.add(request) + end + + context "when expectation found" do + it "calls on_headers callbacks" do + canary = :not_called + request.on_headers do + canary = :called + end + hydra.add(request) + hydra.run + expect(canary).to eq(:called) + end + + it "calls on_body callbacks" do + canary = :not_called + request.on_body do + canary = :called + end + hydra.add(request) + hydra.run + expect(canary).to eq(:called) + end + + it "finishes response" do + expect(request).to receive(:finish) + hydra.add(request) + end + + it "is a mock" do + hydra.add(request) + expect(request.response.mock).to be(true) + end + end + end +end diff --git a/spec/typhoeus/hydra_spec.rb b/spec/typhoeus/hydra_spec.rb new file mode 100644 index 0000000..a8a35aa --- /dev/null +++ b/spec/typhoeus/hydra_spec.rb @@ -0,0 +1,22 @@ +require File.expand_path(File.dirname(__FILE__) + '/../spec_helper') + +describe Typhoeus::Hydra do + let(:base_url) { "localhost:3001" } + let(:options) { {} } + let(:hydra) { Typhoeus::Hydra.new(options) } + + describe "#new" do + let(:options) { {:pipeling => true} } + + it "passes options to multi" do + expect(Ethon::Multi).to receive(:new).with(options) + hydra + end + end + + describe "#hydra" do + it "returns a hydra" do + expect(Typhoeus::Hydra.hydra).to be_a(Typhoeus::Hydra) + end + end +end diff --git a/spec/typhoeus/pool_spec.rb b/spec/typhoeus/pool_spec.rb new file mode 100644 index 0000000..b04f433 --- /dev/null +++ b/spec/typhoeus/pool_spec.rb @@ -0,0 +1,137 @@ +require 'spec_helper' + +describe Typhoeus::Pool do + let(:easy) { Ethon::Easy.new } + after { Typhoeus::Pool.clear } + + describe "#easies" do + it "returns array" do + expect(Typhoeus::Pool.send(:easies)).to be_a(Array) + end + end + + describe "#release" do + it "resets easy" do + expect(easy).to receive(:reset) + Typhoeus::Pool.release(easy) + end + + it "flush cookies to disk" do + expect(easy).to receive(:cookielist=).with('flush') + expect(easy).to receive(:reset) + expect(easy).to receive(:cookielist=).with('all') + Typhoeus::Pool.release(easy) + end + + it "writes cookies to disk" do + tempfile1 = Tempfile.new('cookies') + tempfile2 = Tempfile.new('cookies') + + easy.cookiejar = tempfile1.path + easy.url = "localhost:3001/cookies-test" + easy.perform + + Typhoeus::Pool.release(easy) + + expect(File.zero?(tempfile1.path)).to be(false) + expect(File.read(tempfile1.path)).to match(/\s+foo\s+bar$/) + expect(File.read(tempfile1.path)).to match(/\s+bar\s+foo$/) + + # do it again - and check if tempfile1 wasn't change + easy.cookiejar = tempfile2.path + easy.url = "localhost:3001/cookies-test2" + easy.perform + + Typhoeus::Pool.release(easy) + + # tempfile 1 + expect(File.zero?(tempfile1.path)).to be(false) + expect(File.read(tempfile1.path)).to match(/\s+foo\s+bar$/) + expect(File.read(tempfile1.path)).to match(/\s+bar\s+foo$/) + + # tempfile2 + expect(File.zero?(tempfile2.path)).to be(false) + expect(File.read(tempfile2.path)).to match(/\s+foo2\s+bar$/) + expect(File.read(tempfile2.path)).to match(/\s+bar2\s+foo$/) + end + + it "puts easy back into pool" do + Typhoeus::Pool.release(easy) + expect(Typhoeus::Pool.send(:easies)).to include(easy) + end + + context "when threaded access" do + it "releases correct number of easies" do + (0..9).map do |n| + Thread.new do + Typhoeus::Pool.release(Ethon::Easy.new) + end + end.map(&:join) + expect(Typhoeus::Pool.send(:easies).size).to eq(10) + end + end + end + + describe "#get" do + context "when easy in pool" do + before { Typhoeus::Pool.send(:easies) << easy } + + it "takes" do + expect(Typhoeus::Pool.get).to eq(easy) + end + end + + context "when no easy in pool" do + it "creates" do + expect(Typhoeus::Pool.get).to be_a(Ethon::Easy) + end + + context "when threaded access" do + it "creates correct number of easies" do + queue = Queue.new + (0..9).map do |n| + Thread.new do + queue.enq(Typhoeus::Pool.get) + end + end.map(&:join) + + array = Array.new(queue.size) { queue.pop } + expect(array.uniq.size).to eq(10) + end + end + end + + context "when forked" do + before do + allow(Process).to receive(:pid).and_return(1) + Typhoeus::Pool.send(:easies) << easy + allow(Process).to receive(:pid).and_return(2) + end + + after do + allow(Process).to receive(:pid).and_call_original + Typhoeus::Pool.instance_variable_set(:@pid, Process.pid) + end + + it "creates" do + expect(Typhoeus::Pool.get).to_not eq(easy) + end + end + end + + describe "#with" do + it "is re-entrant" do + array = [] + Typhoeus::Pool.with_easy do |e1| + array << e1 + Typhoeus::Pool.with_easy do |e2| + array << e2 + Typhoeus::Pool.with_easy do |e3| + array << e3 + end + end + end + expect(array.uniq.size).to eq(3) + end + end +end diff --git a/spec/typhoeus/request/actions_spec.rb b/spec/typhoeus/request/actions_spec.rb new file mode 100644 index 0000000..03493e1 --- /dev/null +++ b/spec/typhoeus/request/actions_spec.rb @@ -0,0 +1,19 @@ +require 'spec_helper' + +describe Typhoeus::Request::Actions do + [:get, :post, :put, :delete, :head, :patch, :options].each do |name| + describe ".#{name}" do + let(:response) { Typhoeus::Request.method(name).call("http://localhost:3001") } + + it "returns ok" do + expect(response.return_code).to eq(:ok) + end + + unless name == :head + it "makes #{name.to_s.upcase} Request" do + expect(response.response_body).to include("\"REQUEST_METHOD\":\"#{name.to_s.upcase}\"") + end + end + end + end +end diff --git a/spec/typhoeus/request/before_spec.rb b/spec/typhoeus/request/before_spec.rb new file mode 100644 index 0000000..ed39b0c --- /dev/null +++ b/spec/typhoeus/request/before_spec.rb @@ -0,0 +1,93 @@ +require 'spec_helper' + +describe Typhoeus::Request::Before do + let(:request) { Typhoeus::Request.new("") } + let(:receive_counter) { double :mark => :twain } + + describe "#queue" do + context "when before" do + context "when one" do + it "executes" do + Typhoeus.before { |r| receive_counter.mark } + expect(receive_counter).to receive(:mark) + request.run + end + + context "when true" do + it "calls super" do + Typhoeus.before { true } + expect(Typhoeus::Expectation).to receive(:response_for) + request.run + end + end + + context "when false" do + it "doesn't call super" do + Typhoeus.before { false } + expect(Typhoeus::Expectation).to receive(:response_for).never + request.run + end + + it "returns response" do + Typhoeus.before { |r| r.response = 1; false } + expect(request.run).to be(1) + end + end + + context "when a response" do + it "doesn't call super" do + Typhoeus.before { Typhoeus::Response.new } + expect(Typhoeus::Expectation).to receive(:response_for).never + request.run + end + + it "returns response" do + Typhoeus.before { |r| r.response = Typhoeus::Response.new } + expect(request.run).to be_a(Typhoeus::Response) + end + end + end + + context "when multi" do + context "when all true" do + before { 3.times { Typhoeus.before { |r| receive_counter.mark } } } + + it "calls super" do + expect(Typhoeus::Expectation).to receive(:response_for) + request.run + end + + it "executes all" do + expect(receive_counter).to receive(:mark).exactly(3) + request.run + end + end + + context "when middle false" do + before do + Typhoeus.before { |r| receive_counter.mark } + Typhoeus.before { |r| receive_counter.mark; nil } + Typhoeus.before { |r| receive_counter.mark } + end + + it "doesn't call super" do + expect(Typhoeus::Expectation).to receive(:response_for).never + request.run + end + + it "executes only two" do + expect(receive_counter).to receive(:mark).exactly(2).times + request.run + end + end + end + end + + context "when no before" do + it "calls super" do + expect(Typhoeus::Expectation).to receive(:response_for) + request.run + end + end + end +end diff --git a/spec/typhoeus/request/block_connection_spec.rb b/spec/typhoeus/request/block_connection_spec.rb new file mode 100644 index 0000000..2ddacdc --- /dev/null +++ b/spec/typhoeus/request/block_connection_spec.rb @@ -0,0 +1,75 @@ +require 'spec_helper' + +describe Typhoeus::Request::BlockConnection do + let(:base_url) { "localhost:3001" } + let(:request) { Typhoeus::Request.new(base_url, {:method => :get}) } + + describe "run" do + context "when blocked" do + before { request.block_connection = true } + + it "raises" do + expect{ request.run }.to raise_error(Typhoeus::Errors::NoStub) + end + end + + context "when not blocked" do + before { request.block_connection = false } + + it "doesn't raise" do + expect{ request.run }.to_not raise_error + end + end + end + + describe "#blocked?" do + context "when local block_connection" do + context "when true" do + before { request.block_connection = true } + + it "returns true" do + expect(request.blocked?).to be_truthy + end + end + + context "when false" do + before { request.block_connection = false } + + it "returns false" do + expect(request.blocked?).to be_falsey + end + end + end + + context "when global block_connection" do + context "when true" do + before { Typhoeus::Config.block_connection = true } + after { Typhoeus::Config.block_connection = false } + + it "returns true" do + expect(request.blocked?).to be_truthy + end + end + + context "when false" do + before { Typhoeus::Config.block_connection = false } + + it "returns false" do + expect(request.blocked?).to be_falsey + end + end + end + + context "when global and local block_connection" do + before do + Typhoeus::Config.block_connection = true + request.block_connection = false + end + after { Typhoeus::Config.block_connection = false } + + it "takes local" do + expect(request.blocked?).to be_falsey + end + end + end +end diff --git a/spec/typhoeus/request/cacheable_spec.rb b/spec/typhoeus/request/cacheable_spec.rb new file mode 100644 index 0000000..bffa3e4 --- /dev/null +++ b/spec/typhoeus/request/cacheable_spec.rb @@ -0,0 +1,94 @@ +require 'spec_helper' + +describe Typhoeus::Request::Cacheable do + let(:cache) { MemoryCache.new } + let(:options) { {} } + let(:request) { Typhoeus::Request.new("http://localhost:3001", options) } + let(:response) { Typhoeus::Response.new } + + before { Typhoeus::Config.cache = cache } + after { Typhoeus::Config.cache = false } + + describe "#response=" do + context "when cache activated" do + context "when request new" do + it "caches response" do + request.response = response + expect(cache.memory[request]).to be + end + + it "doesn't set cached on response" do + request.response = response + expect(request.response.cached?).to be_falsey + end + end + + context "when request in memory" do + before { cache.memory[request] = response } + + it "finishes request" do + expect(request).to receive(:finish).with(response) + request.run + end + + it "sets cached to true for response" do + request.run + expect(request.response.cached?).to be_truthy + end + end + end + end + + describe "#run" do + context "when cache activated" do + context "when request new" do + it "fetches response" do + expect(request.response).to_not be(response) + end + end + + context "when request in memory" do + before { cache.memory[request] = response } + + it "finishes request" do + expect(request).to receive(:finish).with(response) + request.run + end + end + + context "when cache is specified on a request" do + before { Typhoeus::Config.cache = false } + + context "when cache is false" do + let(:options) { { :cache => false } } + + it "finishes request" do + expect(request.response).to_not be(response) + request.run + end + end + + context "when cache is defined" do + let(:options) { { :cache => cache } } + + before { cache.memory[request] = response } + + it "finishes request" do + expect(request).to receive(:finish).with(response) + request.run + end + end + end + end + end + + describe "#cache_ttl" do + context "when option[:cache_ttl]" do + let(:options) { {:cache_ttl => 1} } + + it "returns" do + expect(request.cache_ttl).to be(1) + end + end + end +end diff --git a/spec/typhoeus/request/callbacks_spec.rb b/spec/typhoeus/request/callbacks_spec.rb new file mode 100644 index 0000000..9b30e74 --- /dev/null +++ b/spec/typhoeus/request/callbacks_spec.rb @@ -0,0 +1,91 @@ +require 'spec_helper' + +describe Typhoeus::Request::Callbacks do + let(:request) { Typhoeus::Request.new("fubar") } + + [:on_complete, :on_success, :on_failure, :on_progress].each do |callback| + describe "##{callback}" do + it "responds" do + expect(request).to respond_to(callback) + end + + context "when no block given" do + it "returns @#{callback}" do + expect(request.method(callback).call).to eq([]) + end + end + + context "when block given" do + it "stores" do + request.method(callback).call { p 1 } + expect(request.instance_variable_get("@#{callback}").size).to eq(1) + end + end + + context "when multiple blocks given" do + it "stores" do + request.method(callback).call { p 1 } + request.method(callback).call { p 2 } + expect(request.instance_variable_get("@#{callback}").size).to eq(2) + end + end + end + end + + describe "#execute_callbacks" do + [:on_complete, :on_success, :on_failure, :on_progress].each do |callback| + context "when #{callback}" do + context "when local callback" do + before do + code = if callback == :on_failure + 500 + else + 200 + end + request.response = Typhoeus::Response.new(:mock => true, :response_code => code) + request.method(callback).call {|r| expect(r).to be_a(Typhoeus::Response) } + end + + it "executes blocks and passes response" do + request.execute_callbacks + end + + it "sets handled_response" do + request.method(callback).call { 1 } + request.execute_callbacks + expect(request.response.handled_response).to be(1) + end + end + + context "when global callback" do + before do + request.response = Typhoeus::Response.new + Typhoeus.method(callback).call {|r| expect(r).to be_a(Typhoeus::Response) } + end + + it "executes blocks and passes response" do + request.execute_callbacks + end + end + + context "when global and local callbacks" do + before do + request.response = Typhoeus::Response.new + Typhoeus.method(callback).call {|r| r.instance_variable_set(:@fu, 1) } + request.method(callback).call {|r| expect(r.instance_variable_get(:@fu)).to eq(1) } + end + + it "runs global first" do + request.execute_callbacks + end + end + end + end + + context "when local on_complete and gobal on_success" do + it "runs all global callbacks first" do + skip + end + end + end +end diff --git a/spec/typhoeus/request/marshal_spec.rb b/spec/typhoeus/request/marshal_spec.rb new file mode 100644 index 0000000..6ab922a --- /dev/null +++ b/spec/typhoeus/request/marshal_spec.rb @@ -0,0 +1,60 @@ +require 'spec_helper' + +describe Typhoeus::Request::Marshal do + let(:base_url) { "localhost:3001" } + let(:request) { Typhoeus::Request.new(base_url) } + + describe "#marshal_dump" do + %w(on_complete on_success on_failure on_progress).each do |name| + context "when #{name} handler" do + before { request.instance_variable_set("@#{name}", Proc.new{}) } + + it "doesn't include @#{name}" do + expect(request.send(:marshal_dump).map(&:first)).to_not include("@#{name}") + end + + it "doesn't raise when dumped" do + expect { Marshal.dump(request) }.to_not raise_error + end + + context "when loading" do + let(:loaded) { Marshal.load(Marshal.dump(request)) } + + it "includes base_url" do + expect(loaded.base_url).to eq(request.base_url) + end + + it "doesn't include #{name}" do + expect(loaded.instance_variables).to_not include("@#{name}") + end + end + end + end + + context 'when run through hydra' do + let(:options) { {} } + let(:hydra) { Typhoeus::Hydra.new(options) } + + before(:each) do + hydra.queue(request) + hydra.run + end + + it "doesn't include @hydra" do + expect(request.send(:marshal_dump).map(&:first)).to_not include("@hydra") + end + + context 'when loading' do + let(:loaded) { Marshal.load(Marshal.dump(request)) } + + it "includes base_url" do + expect(loaded.base_url).to eq(request.base_url) + end + + it "doesn't include #{name}" do + expect(loaded.instance_variables).to_not include("@hydra") + end + end + end + end +end diff --git a/spec/typhoeus/request/memoizable_spec.rb b/spec/typhoeus/request/memoizable_spec.rb new file mode 100644 index 0000000..1bb7ebc --- /dev/null +++ b/spec/typhoeus/request/memoizable_spec.rb @@ -0,0 +1,34 @@ +require 'spec_helper' + +describe Typhoeus::Request::Memoizable do + let(:options) { {} } + let(:request) { Typhoeus::Request.new("fu", options) } + let(:response) { Typhoeus::Response.new } + let(:hydra) { Typhoeus::Hydra.new } + + describe "#response=" do + context "when memoization activated" do + before { Typhoeus::Config.memoize = true } + after { Typhoeus::Config.memoize = false } + + context "when GET request" do + let(:options) { {:method => :get} } + before { request.hydra = hydra } + + it "stores response in memory" do + request.response = response + expect(hydra.memory[request]).to be + end + end + + context "when no GET request" do + let(:options) { {:method => :post} } + + it "doesn't store response in memory" do + request.response = response + expect(hydra.memory[request]).to be_nil + end + end + end + end +end diff --git a/spec/typhoeus/request/operations_spec.rb b/spec/typhoeus/request/operations_spec.rb new file mode 100644 index 0000000..79f6be0 --- /dev/null +++ b/spec/typhoeus/request/operations_spec.rb @@ -0,0 +1,101 @@ +require 'spec_helper' + +describe Typhoeus::Request::Operations do + let(:base_url) { "localhost:3001" } + let(:options) { {} } + let(:request) { Typhoeus::Request.new(base_url, options) } + + describe "#run" do + let(:easy) { Ethon::Easy.new } + before { expect(Typhoeus::Pool).to receive(:get).and_return(easy) } + + it "grabs an easy" do + request.run + end + + it "generates settings" do + expect(easy).to receive(:http_request) + request.run + end + + it "performs" do + expect(easy).to receive(:perform) + request.run + end + + it "sets response" do + request.run + expect(request.response).to be + end + + it "releases easy" do + expect(Typhoeus::Pool).to receive(:release) + request.run + end + + it "calls on_body" do + on_body_called = false + request.on_body { |body, response| on_body_called = true } + request.run + expect(on_body_called).to be_truthy + expect(request.response.body).to satisfy { |v| v.nil? || v == '' } + end + + it "makes response headers available to on_body" do + headers = nil + request.on_body { |body, response| headers = response.headers } + request.run + expect(headers).to be + expect(headers).to eq(request.response.headers) + end + + it "calls on_headers and on_body" do + headers = nil + request.on_headers { |response| headers = response.headers } + request.on_body { |body, response| expect(headers).not_to be_nil ; expect(response.headers).to eq(headers) } + request.on_complete { |response| expect(response).not_to be_nil ; expect(response.headers).to eq(headers) ; expect(response.body).to be_empty } + request.run + end + + it "calls on_headers and on_complete" do + headers = nil + request.on_headers { |response| headers = response.headers } + request.on_complete { |response| expect(response).not_to be_nil ; expect(response.headers).to eq(headers) ; expect(response.body).not_to be_empty } + request.run + end + + it "calls on_complete" do + callback = double(:call) + expect(callback).to receive(:call) + request.instance_variable_set(:@on_complete, [callback]) + request.run + end + + it "returns a response" do + expect(request.run).to be_a(Typhoeus::Response) + end + end + + describe "#finish" do + let(:response) { Typhoeus::Response.new } + + it "assigns response" do + request.finish(response) + expect(request.response).to be(response) + end + + it "assigns request to response" do + request.finish(response) + expect(request.response.request).to be(request) + end + + it "executes callbacks" do + expect(request).to receive(:execute_callbacks) + request.finish(response) + end + + it "returns response" do + expect(request.finish(response)).to be(response) + end + end +end diff --git a/spec/typhoeus/request/responseable_spec.rb b/spec/typhoeus/request/responseable_spec.rb new file mode 100644 index 0000000..01087bc --- /dev/null +++ b/spec/typhoeus/request/responseable_spec.rb @@ -0,0 +1,13 @@ +require 'spec_helper' + +describe Typhoeus::Request::Responseable do + let(:request) { Typhoeus::Request.new("base_url", {}) } + let(:response) { Typhoeus::Response.new } + + describe "#response=" do + it "stores response" do + request.response = response + expect(request.response).to eq(response) + end + end +end diff --git a/spec/typhoeus/request/stubbable_spec.rb b/spec/typhoeus/request/stubbable_spec.rb new file mode 100644 index 0000000..4651d5f --- /dev/null +++ b/spec/typhoeus/request/stubbable_spec.rb @@ -0,0 +1,45 @@ +require 'spec_helper' + +describe Typhoeus::Request::Stubbable do + let(:base_url) { "localhost:3001" } + let(:request) { Typhoeus::Request.new(base_url) } + let(:response) { Typhoeus::Response.new } + + before { Typhoeus.stub(base_url).and_return(response) } + + describe "#run" do + it "checks expectations" do + request.run + end + + context "when expectation found" do + it "calls on_headers callbacks" do + canary = :not_called + request.on_headers do + canary = :called + end + request.run + expect(canary).to eq(:called) + end + + it "calls on_body callbacks" do + canary = :not_called + request.on_body do + canary = :called + end + request.run + expect(canary).to eq(:called) + end + + it "finishes request" do + expect(request).to receive(:finish) + request.run + end + + it "sets mock on response" do + request.run + expect(request.response.mock).to be(true) + end + end + end +end diff --git a/spec/typhoeus/request_spec.rb b/spec/typhoeus/request_spec.rb new file mode 100644 index 0000000..cabcb4c --- /dev/null +++ b/spec/typhoeus/request_spec.rb @@ -0,0 +1,232 @@ +require 'spec_helper' + +describe Typhoeus::Request do + let(:base_url) { "localhost:3001" } + let(:options) { {:verbose => true, :headers => { 'User-Agent' => "Fubar", 'Expect' => "" }, :maxredirs => 50} } + let(:request) { Typhoeus::Request.new(base_url, options) } + + describe ".url" do + context "when no parameters" do + it "returns base_url" do + expect(request.url).to eq(request.base_url) + end + end + + context "when parameters" do + let(:options) { {:params => {:a => 1}} } + + it "returns full url" do + expect(request.url).to eq("#{request.base_url}?a=1") + end + end + + it "pushes an easy back into the pool" do + easy = double.as_null_object + allow(Typhoeus::Pool).to receive(:get).and_return(easy) + expect(Typhoeus::Pool).to receive(:release).with(easy) + request.url + end + end + + describe ".new" do + it "stores base_url" do + expect(request.base_url).to eq(base_url) + end + + it "stores options" do + expect(request.options).to eq(options) + end + + it "stores original options" do + expect(request.original_options).to eq(options) + expect(request.original_options).to_not be(request.options) + end + + it "sets defaults" do + expect(request.options[:headers]['User-Agent']).to be + end + end + + describe "set_defaults" do + context "when header with user agent" do + let(:options) { {:headers => {'User-Agent' => "Custom"} } } + + it "doesn't modify user agent" do + expect(request.options[:headers]['User-Agent']).to eq("Custom") + end + end + + context "when header without user agent" do + let(:options) { {:headers => {} } } + + it "add user agent" do + agent = request.options[:headers]['User-Agent'] + expect(agent).to eq(Typhoeus::USER_AGENT) + end + end + + context "when Config.user_agent set" do + before { Typhoeus.configure { |config| config.user_agent = "Default" } } + after { Typhoeus.configure { |config| config.user_agent = nil } } + + context "with headers" do + let(:options) { {:headers => { "User-Agent" => "Fubar" } } } + + it "uses the request options' user agent" do + expect(request.options[:headers]["User-Agent"]).to eq("Fubar") + end + end + + context "without headers" do + let(:options) { {:headers => {} } } + + it "uses the global options' user agent" do + expect(request.options[:headers]["User-Agent"]).to eq("Default") + end + end + end + + context "when Config.verbose set" do + before { Typhoeus.configure { |config| config.verbose = true} } + after { Typhoeus.configure { |config| config.verbose = false} } + + it "respects" do + expect(request.options[:verbose]).to be_truthy + end + end + + context "when maxredirs" do + context "when not set" do + it "defaults to 50" do + expect(request.options[:maxredirs]).to be(50) + end + end + + context "when set" do + let(:options) { {:maxredirs => 1} } + + it "respects" do + expect(request.options[:maxredirs]).to be(1) + end + end + end + + context "when Config.proxy set" do + before { Typhoeus.configure { |config| config.proxy = "http://proxy.internal" } } + after { Typhoeus.configure { |config| config.proxy = nil } } + + it "respects" do + expect(request.options[:proxy]).to eq("http://proxy.internal") + end + + context "when option proxy set" do + let(:options) { {:proxy => nil} } + + it "does not override" do + expect(request.options[:proxy]).to be_nil + end + end + end + end + + describe "#eql?" do + context "when another class" do + let(:other) { "" } + + it "returns false" do + expect(request).to_not eql other + end + end + + context "when same class" do + let(:other) { Typhoeus::Request.new("base_url", options) } + + context "when other base_url" do + it "returns false" do + expect(request).to_not eql other + end + end + + context "when same base_url and other options" do + let(:other) { Typhoeus::Request.new(base_url, {}) } + + it "returns false" do + expect(request).to_not eql other + end + end + + + context "when same base_url and options" do + context "when same order" do + let(:other) { Typhoeus::Request.new(base_url, options) } + + it "returns true" do + expect(request).to eql other + end + end + + context "when different order" do + let(:other_options) { + {:headers => { 'User-Agent' => "Fubar", 'Expect' => ""}, :verbose => true } + } + let(:other) { Typhoeus::Request.new(base_url, other_options)} + + it "returns true" do + expect(request).to eql other + end + end + end + end + end + + describe "#hash" do + context "when request.eql?(other)" do + context "when different order" do + let(:other_options) { + {:headers => { 'User-Agent' => "Fubar", 'Expect' => "" }, :verbose => true } + } + let(:other) { Typhoeus::Request.new(base_url, other_options)} + + it "has same hashes" do + expect(request.hash).to eq(other.hash) + end + end + + context "when same order" do + let(:other) { Typhoeus::Request.new(base_url, options) } + + it "has same hashes" do + expect(request.hash).to eq(other.hash) + end + end + + context "when hashes with different orders are contained in arrays" do + let(:request) { Typhoeus::Request.new(base_url, :params => [{:b => 2, :a => 1}]) } + let(:other) { Typhoeus::Request.new(base_url, :params => [{:a => 1, :b => 2}]) } + it "has different hashes" do + expect(request.hash).to eq(other.hash) + end + end + end + + context "when not request.eql?(other)" do + let(:request) { Typhoeus::Request.new(base_url, :params => {:foo => 'bar'}) } + let(:other) { Typhoeus::Request.new(base_url, :params => {:foo => 'baz'}) } + + it "has different hashes" do + expect(request.hash).to_not eq(other.hash) + end + end + end + + describe "#encoded_body" do + let(:request) { + Typhoeus::Request.new("www.example.com",:body => {:a => 1}) + } + + it "returns encoded body" do + expect(request.encoded_body).to eq("a=1") + end + end + +end diff --git a/spec/typhoeus/response/header_spec.rb b/spec/typhoeus/response/header_spec.rb new file mode 100644 index 0000000..922d99e --- /dev/null +++ b/spec/typhoeus/response/header_spec.rb @@ -0,0 +1,147 @@ +require 'spec_helper' + +describe Typhoeus::Response::Header do + let(:raw) { nil } + let(:header) { Typhoeus::Response::Header.new(raw) } + + describe ".new" do + context "when string" do + let(:raw) { 'Date: Fri, 29 Jun 2012 10:09:23 GMT' } + + it "sets Date" do + expect(header['Date']).to eq('Fri, 29 Jun 2012 10:09:23 GMT') + end + + it "provides case insensitive access" do + expect(header['DaTe']).to eq('Fri, 29 Jun 2012 10:09:23 GMT') + end + + it "provides symbol access" do + expect(header[:date]).to eq('Fri, 29 Jun 2012 10:09:23 GMT') + end + end + + context "when hash" do + let(:raw) { { 'Date' => 'Fri, 29 Jun 2012 10:09:23 GMT' } } + + it "sets Date" do + expect(header['Date']).to eq(raw['Date']) + end + + it "provides case insensitive access" do + expect(header['DaTe']).to eq(raw['Date']) + end + end + end + + describe "#parse" do + context "when no header" do + it "returns nil" do + expect(header).to be_empty + end + end + + context "when header" do + let(:raw) do + 'HTTP/1.1 200 OK + Set-Cookie: NID=61=LblqYgUOu; expires=Sat, 29-Dec-2012 10:09:23 GMT; path=/; domain=.google.de; HttpOnly + Date: Fri, 29 Jun 2012 10:09:23 GMT + Expires: -1 + Cache-Control: private, max-age=0 + Content-Type: text/html; charset=ISO-8859-1 + Set-Cookie: PREF=ID=77e93yv0hPtejLou; expires=Sun, 29-Jun-2014 10:09:23 GMT; path=/; domain=.google.de + Set-Cookie: NID=61=LblqYgh5Ou; expires=Sat, 29-Dec-2012 10:09:23 GMT; path=/; domain=.google.de; HttpOnly + P3P: CP="This is not a P3P policy! See http://www.google.com/support/accounts/bin/answer.py?hl=en&answer=151657 for more info." + Server: gws + X-XSS-Protection: 1; mode=block + X-Frame-Options: SAMEORIGIN + Transfer-Encoding: chunked'.gsub(/^\s{8}/, '') + end + + it "sets raw" do + expect(header.send(:raw)).to eq(raw) + end + + it "sets Set-Cookie" do + expect(header['set-cookie'].size).to eq(3) + end + + it "provides case insensitive access" do + expect(header['Set-CooKie'].size).to eq(3) + end + + [ + 'NID=61=LblqYgUOu; expires=Sat, 29-Dec-2012 10:09:23 GMT; path=/; domain=.google.de; HttpOnly', + 'PREF=ID=77e93yv0hPtejLou; expires=Sun, 29-Jun-2014 10:09:23 GMT; path=/; domain=.google.de', + 'NID=61=LblqYgh5Ou; expires=Sat, 29-Dec-2012 10:09:23 GMT; path=/; domain=.google.de; HttpOnly' + ].each_with_index do |cookie, i| + it "sets Cookie##{i}" do + expect(header['set-cookie']).to include(cookie) + end + end + + { + 'Date' => 'Fri, 29 Jun 2012 10:09:23 GMT', 'Expires' => '-1', + 'Cache-Control' => 'private, max-age=0', + 'Content-Type' => 'text/html; charset=ISO-8859-1', + 'P3P' => 'CP="This is not a P3P policy! See http://www.google.com/support/accounts/bin/answer.py?hl=en&answer=151657 for more info."', + 'Server' => 'gws', 'X-XSS-Protection' => '1; mode=block', + 'X-Frame-Options' => 'SAMEORIGIN', 'Transfer-Encoding' => 'chunked' + }.each do |name, value| + it "sets #{name}" do + expect(header[name.downcase]).to eq(value) + end + end + + context 'includes a multi-line header' do + let(:raw) do + 'HTTP/1.1 200 OK + Date: Fri, 29 Jun 2012 10:09:23 GMT + Content-Security-Policy: default-src "self"; + img-src * data: "self"; + upgrade-insecure-requests;'.gsub(/^\s{10}/, '') + end + + it "joins header parts" do + expect(header).to eq({ + 'Date' => 'Fri, 29 Jun 2012 10:09:23 GMT', + 'Content-Security-Policy' => 'default-src "self"; img-src * data: "self"; upgrade-insecure-requests;' + }) + end + end + + context 'includes line with only whitespace' do + let(:raw) do + 'HTTP/1.1 200 OK + Date: Fri, 29 Jun 2012 10:09:23 GMT + + '.gsub(/^\s{10}/, '') + end + + it 'ignores it' do + expect(header).to eq({ 'Date' => 'Fri, 29 Jun 2012 10:09:23 GMT' }) + end + end + + context 'with broken headers' do + let(:raw) do + 'HTTP/1.1 200 OK + Date: + Content-Type + '.gsub(/^\s{10}/, '') + end + + it 'returns empty string for invalid headers' do + expect(header.to_hash).to include({ 'Date' => '', 'Content-Type' => '' }) + end + end + end + end + + it "can be Marshal'd" do + header = Typhoeus::Response::Header.new("Foo: Bar") + expect { + Marshal.dump(header) + }.not_to raise_error + end +end diff --git a/spec/typhoeus/response/informations_spec.rb b/spec/typhoeus/response/informations_spec.rb new file mode 100644 index 0000000..896931a --- /dev/null +++ b/spec/typhoeus/response/informations_spec.rb @@ -0,0 +1,283 @@ +require 'spec_helper' + +describe Typhoeus::Response::Informations do + let(:options) { {} } + let(:response) { Typhoeus::Response.new(options) } + + describe "#return_code" do + let(:options) { { :return_code => :ok } } + + it "returns return_code from options" do + expect(response.return_code).to be(:ok) + end + end + + describe "#debug_info" do + let(:options) { { :debug_info => Ethon::Easy::DebugInfo.new } } + + it "returns debug_info from options" do + expect(response.debug_info).to be_a(Ethon::Easy::DebugInfo) + end + end + + describe "#return_message" do + let(:options) { { :return_code => :couldnt_connect } } + + it "returns a message" do + expect(response.return_message).to eq("Couldn't connect to server") + end + + describe "with nil return_code" do + let(:options) { { :return_code => nil } } + + it "returns nil" do + expect(response.return_message).to be_nil + end + end + end + + describe "#response_body" do + context "when response_body" do + let(:options) { { :response_body => "body" } } + + it "returns response_body from options" do + expect(response.response_body).to eq("body") + end + end + + context "when body" do + let(:options) { { :body => "body" } } + + it "returns body from options" do + expect(response.body).to eq("body") + end + end + end + + describe "#response_headers" do + let(:options) { { :response_headers => "Length: 1" } } + + context "when no mock" do + it "returns response_headers from options" do + expect(response.response_headers).to eq("Length: 1") + end + end + + context "when mock" do + context "when no response_headers" do + context "when headers" do + let(:options) { { :mock => true, :headers => {"Length" => 1, "Content-Type" => "text/plain" } } } + + it "constructs response_headers" do + expect(response.response_headers).to include("Length: 1") + expect(response.response_headers).to include("Content-Type: text/plain") + expect(response.response_headers).to include("\r\n") + end + end + + context "when multiple values for a header" do + let(:options) { { :mock => true, :headers => {"Length" => 1, "Content-Type" => "text/plain", "set-cookie" => ["cookieone=one","cookietwo=two"] } } } + + it "constructs response_headers" do + expect(response.response_headers).to include("Length: 1") + expect(response.response_headers).to include("Content-Type: text/plain") + expect(response.response_headers).to include("set-cookie: cookieone=one,cookietwo=two") + expect(response.response_headers).to include("\r\n") + end + end + end + end + end + + describe "#response_code" do + context "when response_code" do + let(:options) { { :response_code => "200" } } + + it "returns response_code from options" do + expect(response.response_code).to eq(200) + end + end + + context "when code" do + let(:options) { { :code => "200" } } + + it "returns code from options" do + expect(response.code).to eq(200) + end + end + end + + describe "#httpauth_avail" do + let(:options) { { :httpauth_avail => "code" } } + + it "returns httpauth_avail from options" do + expect(response.httpauth_avail).to eq("code") + end + end + + describe "#total_time" do + let(:options) { { :total_time => 1 } } + + it "returns total_time from options" do + expect(response.total_time).to eq(1) + end + end + + describe "#starttransfer_time" do + let(:options) { { :starttransfer_time => 1 } } + + it "returns starttransfer_time from options" do + expect(response.starttransfer_time).to eq(1) + end + end + + describe "#appconnect_time" do + let(:options) { { :appconnect_time => 1 } } + + it "returns appconnect_time from options" do + expect(response.appconnect_time).to eq(1) + end + end + + describe "#pretransfer_time" do + let(:options) { { :pretransfer_time => 1 } } + + it "returns pretransfer_time from options" do + expect(response.pretransfer_time).to eq(1) + end + end + + describe "#connect_time" do + let(:options) { { :connect_time => 1 } } + + it "returns connect_time from options" do + expect(response.connect_time).to eq(1) + end + end + + describe "#namelookup_time" do + let(:options) { { :namelookup_time => 1 } } + + it "returns namelookup_time from options" do + expect(response.namelookup_time).to eq(1) + end + end + + describe "#redirect_time" do + let(:options) { { :redirect_time => 1 } } + + it "returns redirect_time from options" do + expect(response.redirect_time).to eq(1) + end + end + + describe "#effective_url" do + let(:options) { { :effective_url => "http://www.example.com" } } + + it "returns effective_url from options" do + expect(response.effective_url).to eq("http://www.example.com") + end + end + + describe "#primary_ip" do + let(:options) { { :primary_ip => "127.0.0.1" } } + + it "returns primary_ip from options" do + expect(response.primary_ip).to eq("127.0.0.1") + end + end + + describe "#redirect_count" do + let(:options) { { :redirect_count => 2 } } + + it "returns redirect_count from options" do + expect(response.redirect_count).to eq(2) + end + end + + describe "#request_size" do + let(:options) { { :request_size => 2 } } + + it "returns request_size from options" do + expect(response.request_size).to eq(2) + end + end + + describe "#headers" do + context "when no response_headers" do + it "returns nil" do + expect(response.headers).to be_nil + end + end + + context "when response_headers" do + let(:options) { {:response_headers => "Expire: -1\nServer: gws"} } + + it "returns nonempty headers" do + expect(response.headers).to_not be_empty + end + + it "has Expire" do + expect(response.headers['expire']).to eq('-1') + end + + it "has Server" do + expect(response.headers['server']).to eq('gws') + end + end + + context "when multiple headers" do + let(:options) { {:response_headers => "Server: A\r\n\r\nServer: B"} } + + it "returns the last" do + expect(response.headers['server']).to eq("B") + end + end + + context "when mock" do + context "when headers" do + let(:options) { {:mock => true, :headers => {"Length" => "1"}} } + + it "returns Typhoeus::Response::Header" do + expect(response.headers).to be_a(Typhoeus::Response::Header) + end + + it "returns headers" do + expect(response.headers.to_hash).to include("Length" => "1") + end + end + end + + context "when requesting" do + let(:response) { Typhoeus.get("localhost:3001") } + + it "returns headers" do + expect(response.headers).to_not be_empty + end + end + end + + describe "#redirections" do + context "when no response_headers" do + it "returns empty array" do + expect(response.redirections).to be_empty + end + end + + context "when headers" do + let(:options) { {:response_headers => "Expire: -1\nServer: gws"} } + + it "returns empty array" do + expect(response.redirections).to be_empty + end + end + + context "when multiple headers" do + let(:options) { {:response_headers => "Server: A\r\n\r\nServer: B"} } + + it "returns response from all but last headers" do + expect(response.redirections.size).to eq(1) + end + end + end +end diff --git a/spec/typhoeus/response/status_spec.rb b/spec/typhoeus/response/status_spec.rb new file mode 100644 index 0000000..64ba1e0 --- /dev/null +++ b/spec/typhoeus/response/status_spec.rb @@ -0,0 +1,256 @@ +require 'spec_helper' + +describe Typhoeus::Response::Status do + let(:response) { Typhoeus::Response.new(options) } + let(:options) { {} } + + describe "timed_out?" do + context "when return code is operation_timedout" do + let(:options) { {:return_code => :operation_timedout} } + + it "return true" do + expect(response).to be_timed_out + end + end + end + + describe "#status_message" do + context "when no header" do + it "returns nil" do + expect(response.status_message).to be_nil + end + end + + context "when header" do + context "when no message" do + let(:options) { {:response_headers => "HTTP/1.1 200\r\n"} } + + it "returns nil" do + expect(response.status_message).to be_nil + end + end + + context "when messsage" do + let(:options) { {:response_headers => "HTTP/1.1 200 message\r\n"} } + + it "returns message" do + expect(response.status_message).to eq("message") + end + end + end + end + + describe "#http_version" do + context "when no header" do + it "returns nil" do + expect(response.http_version).to be_nil + end + end + + context "when header" do + context "when no http version" do + let(:options) { {:response_headers => "HTTP OK"} } + + it "returns nil" do + expect(response.http_version).to be_nil + end + end + + context "when invalid http_version" do + let(:options) { {:response_headers => "HTTP foo/bar OK"} } + + it "returns nil" do + expect(response.http_version).to be_nil + end + end + + context "when valid http_version" do + let(:options) { {:response_headers => "HTTP/1.1 OK"} } + + it "returns http_version" do + expect(response.http_version).to eq("1.1") + end + end + end + end + + describe "#success?" do + context "when response code 200-299" do + let(:options) { {:return_code => return_code, :response_code => 201} } + + context "when mock" do + before { response.mock = true } + + context "when return_code :ok" do + let(:return_code) { :ok } + + it "returns true" do + expect(response.success?).to be_truthy + end + end + + context "when return_code nil" do + let(:return_code) { nil } + + it "returns true" do + expect(response.success?).to be_truthy + end + end + end + + context "when no mock" do + before { response.mock = nil } + + context "when return_code :ok" do + let(:return_code) { :ok } + + it "returns true" do + expect(response.success?).to be_truthy + end + end + + context "when return_code nil" do + let(:return_code) { nil } + + it "returns false" do + expect(response.success?).to be_falsey + end + end + end + end + + context "when response code is not 200-299" do + let(:options) { {:return_code => :ok, :response_code => 500} } + + it "returns false" do + expect(response.success?).to be_falsey + end + end + end + + describe "#failure?" do + context "when response code between 300-526 and 100-300" do + let(:options) { {:return_code => return_code, :response_code => 300} } + + context "when mock" do + before { response.mock = true } + + context "when return_code :internal_server_error" do + let(:return_code) { :internal_server_error } + + it "returns true" do + expect(response.failure?).to be_truthy + end + end + + context "when return_code nil" do + let(:return_code) { nil } + + it "returns true" do + expect(response.failure?).to be_truthy + end + end + end + + context "when no mock" do + before { response.mock = nil } + + context "when return_code :internal_server_error" do + let(:return_code) { :internal_server_error } + + it "returns true" do + expect(response.failure?).to be_truthy + end + end + + context "when return_code nil" do + let(:return_code) { nil } + + it "returns false" do + expect(response.failure?).to be_falsey + end + end + end + end + + context "when response code is not 300-526" do + let(:options) { {:return_code => :ok, :response_code => 200} } + + it "returns false" do + expect(response.failure?).to be_falsey + end + end + end + + describe "#modified?" do + context "when response code 304" do + let(:options) { {:return_code => :ok, :response_code => 304} } + + context "when mock" do + before { response.mock = true } + + context "when return_code :ok" do + let(:return_code) { :ok } + + it "returns false" do + expect(response.modified?).to be_falsey + end + end + + context "when return_code nil" do + let(:return_code) { nil } + + it "returns false" do + expect(response.modified?).to be_falsey + end + end + end + + context "when no mock" do + before { response.mock = nil } + + context "when return_code :ok" do + let(:return_code) { :ok } + + it "returns false" do + expect(response.modified?).to be_falsey + end + end + + context "when return_code nil" do + let(:return_code) { nil } + + it "returns true" do + expect(response.modified?).to be_falsey + end + end + end + end + + context "when response code is not 304" do + let(:options) { {:return_code => :ok, :response_code => 500} } + + it "returns true" do + expect(response.modified?).to be_truthy + end + end + end + + describe "#first_header_line" do + context "when multiple header" do + let(:options) { {:response_headers => "1\r\n\r\n2\r\nbla"} } + + it "returns first line of last block" do + expect(response.method(:first_header_line).call).to eq("2") + end + end + + context "when single header" do + let(:options) { {:response_headers => "1"} } + + it "returns first line" do + expect(response.method(:first_header_line).call).to eq("1") + end + end + end +end diff --git a/spec/typhoeus/response_spec.rb b/spec/typhoeus/response_spec.rb new file mode 100644 index 0000000..02512f4 --- /dev/null +++ b/spec/typhoeus/response_spec.rb @@ -0,0 +1,100 @@ +require 'spec_helper' + +describe Typhoeus::Response do + let(:response) { Typhoeus::Response.new(options) } + let(:options) { {} } + + describe ".new" do + context "when options" do + context "when return_code" do + let(:options) { {:return_code => 2} } + + it "stores" do + expect(response.options[:return_code]).to be(2) + end + end + + context "when headers" do + let(:options) { {:headers => {'A' => 'B'}} } + + it "stores unmodified" do + expect(response.options[:headers]).to be(options[:headers]) + end + + it "sets @headers to a Typhoeus::Response::Header" do + expect(response.instance_variable_get(:@headers)).to be_a(Typhoeus::Response::Header) + end + + it "has key" do + expect(response.headers['A']).to eq('B') + end + end + end + end + + describe "#mock" do + context "when @mock" do + before { response.mock = true } + + it "returns @mock" do + expect(response.mock).to be_truthy + end + end + + context "when options[:mock]" do + let(:options) { {:mock => true} } + + it "returns options[:mock]" do + expect(response.mock).to be_truthy + end + end + + context "when @mock and options[:mock]" do + let(:options) { {:mock => 1} } + before { response.mock = 2 } + + it "returns @mock" do + expect(response.mock).to be(2) + end + end + end + + describe "#handled_response" do + let(:handled_response) { Typhoeus::Response.new } + + context "when @handled_response" do + before { response.handled_response = handled_response } + + it "returns @handled_response" do + expect(response.handled_response).to be(handled_response) + end + end + + context "when @handled_response is nil" do + before { response.handled_response = nil } + + it "returns response" do + expect(response.handled_response).to be(response) + end + end + end + + describe "#cached" do + context "when @cached" do + before { response.cached = true } + + it "returns cached status" do + expect(response.cached?).to be_truthy + end + end + + context "when @cached is nil" do + before { response.cached = nil } + + it "returns false" do + expect(response.cached?).to be_falsey + end + end + + end +end diff --git a/spec/typhoeus_spec.rb b/spec/typhoeus_spec.rb new file mode 100644 index 0000000..6432c8d --- /dev/null +++ b/spec/typhoeus_spec.rb @@ -0,0 +1,105 @@ +require 'spec_helper' + +describe Typhoeus do + before(:each) do + Typhoeus.configure { |config| config.verbose = false; config.block_connection = false } + end + + describe ".configure" do + it "yields config" do + Typhoeus.configure do |config| + expect(config).to be_a(Typhoeus::Config) + end + end + + it "sets values config" do + Typhoeus::Config.verbose = true + expect(Typhoeus::Config.verbose).to be_truthy + end + end + + describe ".stub" do + let(:base_url) { "www.example.com" } + + shared_examples "lazy response construction" do + it "calls the block to construct a response when a request matches the stub" do + expected_response = Typhoeus::Response.new + Typhoeus.stub(base_url) do |request| + expected_response + end + + response = Typhoeus.get(base_url) + + expect(response).to be(expected_response) + end + end + + context "when no similar expectation exists" do + include_examples "lazy response construction" + + it "returns expectation" do + expect(Typhoeus.stub(base_url)).to be_a(Typhoeus::Expectation) + end + + it "adds expectation" do + Typhoeus.stub(:get, "") + expect(Typhoeus::Expectation.all.size).to eq(1) + end + end + + context "when similar expectation exists" do + include_examples "lazy response construction" + + let(:expectation) { Typhoeus::Expectation.new(base_url) } + before { Typhoeus::Expectation.all << expectation } + + it "returns expectation" do + expect(Typhoeus.stub(base_url)).to be_a(Typhoeus::Expectation) + end + + it "doesn't add expectation" do + Typhoeus.stub(base_url) + expect(Typhoeus::Expectation.all.size).to eq(1) + end + end + end + + describe ".before" do + it "adds callback" do + Typhoeus.before { true } + expect(Typhoeus.before.size).to eq(1) + end + end + + describe ".with_connection" do + it "executes block with block connection is false" do + Typhoeus.with_connection { expect(Typhoeus::Config.block_connection).to be(false) } + end + + it "sets block connection back to previous value" do + Typhoeus::Config.block_connection = true + Typhoeus.with_connection {} + expect(Typhoeus::Config.block_connection).to be(true) + end + + it "returns result of block" do + expect(Typhoeus.with_connection { "123" }).to eq("123") + end + end + + [:get, :post, :put, :delete, :head, :patch, :options].each do |name| + describe ".#{name}" do + let(:response) { Typhoeus::Request.method(name).call("http://localhost:3001") } + + it "returns ok" do + expect(response.return_code).to eq(:ok) + end + + unless name == :head + it "makes #{name.to_s.upcase} requests" do + expect(response.response_body).to include("\"REQUEST_METHOD\":\"#{name.to_s.upcase}\"") + end + end + end + end +end diff --git a/typhoeus.gemspec b/typhoeus.gemspec new file mode 100644 index 0000000..39bcc7a --- /dev/null +++ b/typhoeus.gemspec @@ -0,0 +1,25 @@ +# encoding: utf-8 +lib = File.expand_path('../lib/', __FILE__) +$:.unshift lib unless $:.include?(lib) + +require 'typhoeus/version' + +Gem::Specification.new do |s| + s.name = "typhoeus" + s.version = Typhoeus::VERSION + s.platform = Gem::Platform::RUBY + s.authors = ["David Balatero", "Paul Dix", "Hans Hasselberg"] + s.email = ["hans.hasselberg@gmail.com"] + s.homepage = "https://github.com/typhoeus/typhoeus" + s.summary = "Parallel HTTP library on top of libcurl multi." + s.description = %q{Like a modern code version of the mythical beast with 100 serpent heads, Typhoeus runs HTTP requests in parallel while cleanly encapsulating handling logic.} + + s.required_rubygems_version = ">= 1.3.6" + s.license = 'MIT' + + s.add_dependency('ethon', [">= 0.9.0"]) + + s.files = `git ls-files`.split("\n") + s.test_files = `git ls-files -- spec/*`.split("\n") + s.require_path = 'lib' +end -- cgit v1.2.3 From 0085ce027d4668e32ed41ccd156aef2a6f03f776 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Boutillier?= Date: Thu, 18 Nov 2021 00:28:31 +0100 Subject: Import ruby-typhoeus_1.4.0-2.debian.tar.xz [dgit import tarball ruby-typhoeus 1.4.0-2 ruby-typhoeus_1.4.0-2.debian.tar.xz] --- changelog | 159 ++++++++++++++++++++++++++++++++++ control | 39 +++++++++ copyright | 33 +++++++ patches/remove-rubygems-bundler.patch | 17 ++++ patches/series | 1 + ruby-tests.rake | 7 ++ ruby-typhoeus.docs | 2 + rules | 20 +++++ salsa-ci.yml | 4 + source/format | 1 + upstream/metadata | 4 + watch | 2 + 12 files changed, 289 insertions(+) create mode 100644 changelog create mode 100644 control create mode 100644 copyright create mode 100644 patches/remove-rubygems-bundler.patch create mode 100644 patches/series create mode 100644 ruby-tests.rake create mode 100644 ruby-typhoeus.docs create mode 100755 rules create mode 100644 salsa-ci.yml create mode 100644 source/format create mode 100644 upstream/metadata create mode 100644 watch diff --git a/changelog b/changelog new file mode 100644 index 0000000..514d3e3 --- /dev/null +++ b/changelog @@ -0,0 +1,159 @@ +ruby-typhoeus (1.4.0-2) unstable; urgency=medium + + * Team upload + + [ Debian Janitor ] + * Remove constraints unnecessary since buster + + [ Jenkins ] + * Update watch file format version to 4. + * Update standards version to 4.5.1, no changes needed. + + [ Cédric Boutillier ] + * Update team name + * Add .gitattributes to keep unwanted files out of the source package + * Bump Standards-Version to 4.6.0 (no changes needed) + * Use gem install layout + * build-depend on ruby-webrick (Closes: #996518) + + -- Cédric Boutillier Thu, 18 Nov 2021 00:28:31 +0100 + +ruby-typhoeus (1.4.0-1) unstable; urgency=medium + + * Team Upload + * New upstream version 1.4.0 + * Fix Control file + * Added Upstream Metadata + + -- Abraham Raji Tue, 30 Jun 2020 10:34:39 +0530 + +ruby-typhoeus (1.4.0~git.20191003.0c66e4e-2) UNRELEASED; urgency=medium + + * Trim trailing whitespace. + * Use secure copyright file specification URI. + * Set upstream metadata fields: Bug-Database, Bug-Submit. + + -- Debian Janitor Mon, 20 Apr 2020 23:53:37 +0000 + +ruby-typhoeus (1.4.0~git.20191003.0c66e4e-1) unstable; urgency=medium + + [ Utkarsh Gupta ] + * Add salsa-ci.yml + + [ Pirate Praveen ] + * New upstream version 1.4.0~git.20191003.0c66e4e + * Bump Standards-Version to 4.5.0 (no changes needed) + * Drop compat file, rely on debhelper-compat and bump compat level to 12 + + -- Pirate Praveen Thu, 06 Feb 2020 12:16:27 +0100 + +ruby-typhoeus (1.3.1-1) unstable; urgency=medium + + * Team upload + * New upstream release + * Standard version updated to 4.2.1 + + -- Manas Kashyap Thu, 13 Dec 2018 06:08:41 +0000 + +ruby-typhoeus (1.3.0-1) unstable; urgency=medium + + * Team upload + * New upstream release + * Bump standards version to 4.1.4 (no changes) + + -- Manas kashyap Sun, 13 May 2018 14:40:59 +0000 + +ruby-typhoeus (1.1.2-1) unstable; urgency=medium + + * Team upload + * New upstream release + + -- Sruthi Chandran Tue, 27 Jun 2017 22:51:15 +0530 + +ruby-typhoeus (1.1.0-1) unstable; urgency=medium + + * Team upload + * New upstream release + + -- Sruthi Chandran Fri, 09 Sep 2016 18:48:01 +0530 + +ruby-typhoeus (1.0.2-1) unstable; urgency=medium + + * New upstream release + * Remove patch fix-specs-for-ruby2_3.patch (merged upstream) + + -- Pirate Praveen Sat, 09 Jul 2016 20:32:36 +0530 + +ruby-typhoeus (0.8.0-2) unstable; urgency=medium + + * Team upload. + + [ Cédric Boutillier ] + * Bump debhelper compatibility level to 9 + * Remove version in the gem2deb build-dependency + * Use https:// in Vcs-* fields + * Use https:// in Vcs-* fields + * Bump Standards-Version to 3.9.7 (no changes needed) + * Run wrap-and-sort on packaging files + + [ Thiago Ribeiro ] + * Revert source code change + * Add patch fix-spec-for-ruby2_3.patch (Closes: #816360) + + -- Thiago Ribeiro Tue, 15 Mar 2016 16:25:53 -0300 + +ruby-typhoeus (0.8.0-1) unstable; urgency=low + + * Team upload + * New upstream release + * Enable Tests + * Check gemspec dependencies during build + + -- Nitesh A Jain Sun, 25 Oct 2015 23:41:45 +0530 + +ruby-typhoeus (0.7.2-2) unstable; urgency=medium + + * Set minimum version of ruby-ethon to 0.7.4 + + -- Pirate Praveen Sat, 29 Aug 2015 23:49:11 +0530 + +ruby-typhoeus (0.7.2-1) unstable; urgency=medium + + * New upstream release + + -- Pirate Praveen Sat, 29 Aug 2015 10:56:27 +0530 + +ruby-typhoeus (0.7.1-2) unstable; urgency=medium + + * Re-upload into unstable + + -- Pirate Praveen Mon, 27 Apr 2015 15:12:29 +0530 + +ruby-typhoeus (0.7.1-1) experimental; urgency=medium + + * New upstream release + + -- Pirate Praveen Thu, 09 Apr 2015 17:30:30 +0530 + +ruby-typhoeus (0.6.9-1) experimental; urgency=medium + + * New upstream release + * Bump standards version to 3.9.6 (no changes) + * Disable tests (some tests need rspec 3) + + -- Pirate Praveen Wed, 19 Nov 2014 00:57:22 +0530 + +ruby-typhoeus (0.6.8-1) unstable; urgency=low + + * New upstream release. + * Bump stadards version to 3.9.5 (no changes) + * Bump gem2deb build dep to >= 0.7.5 (ruby 2.x) + * Bump ruby-ethon dependency to >= 0.7.0 + + -- Pirate Praveen Sun, 13 Apr 2014 17:22:20 +0530 + +ruby-typhoeus (0.6.3-1) unstable; urgency=low + + * Initial release (Closes: #662687) + + -- Praveen Arimbrathodiyil Thu, 16 May 2013 00:34:39 +0530 diff --git a/control b/control new file mode 100644 index 0000000..8399968 --- /dev/null +++ b/control @@ -0,0 +1,39 @@ +Source: ruby-typhoeus +Section: ruby +Priority: optional +Maintainer: Debian Ruby Team +Uploaders: Pirate Praveen +Build-Depends: debhelper-compat (= 13), + gem2deb, + rake, + ruby-ethon, + ruby-faraday, + ruby-json, + ruby-rack, + ruby-rspec, + ruby-sinatra, + ruby-dalli, + ruby-redis, + ruby-webrick (>= 1.7~) +Standards-Version: 4.6.0 +Vcs-Git: https://salsa.debian.org/ruby-team/ruby-typhoeus.git +Vcs-Browser: https://salsa.debian.org/ruby-team/ruby-typhoeus +Homepage: https://github.com/typhoeus/typhoeus +Testsuite: autopkgtest-pkg-ruby +XS-Ruby-Versions: all +Rules-Requires-Root: no + +Package: ruby-typhoeus +Architecture: all +XB-Ruby-Versions: ${ruby:Versions} +Depends: ruby | ruby-interpreter, + ruby-ethon, + ${misc:Depends}, + ${shlibs:Depends} +Description: parallel HTTP library on top of ethon + Typhoeus is a HTTP client library based on Ethon which wraps libcurl. + Sitting on top of libcurl makes Typhoeus very reliable and fast. + . + Like a modern code version of the mythical beast with 100 serpent heads, + Typhoeus runs HTTP requests in parallel while cleanly encapsulating handling + logic. diff --git a/copyright b/copyright new file mode 100644 index 0000000..8e198ff --- /dev/null +++ b/copyright @@ -0,0 +1,33 @@ +Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ +Upstream-Name: typhoeus +Source: http://rubygems.org/gems/typhoeus + +Files: * +Copyright: 2009-2010 Paul Dix + 2011 David Balatero + 2012 Hans Hasselberg +License: Expat + +Files: debian/* +Copyright: 2013 Praveen Arimbrathodiyil +License: Expat + +License: Expat + Permission is hereby granted, free of charge, to any person obtaining + a copy of this software and associated documentation files (the + "Software"), to deal in the Software without restriction, including + without limitation the rights to use, copy, modify, merge, publish, + distribute, sublicense, and/or sell copies of the Software, and to + permit persons to whom the Software is furnished to do so, subject to + the following conditions: + . + The above copyright notice and this permission notice shall be + included in all copies or substantial portions of the Software. + . + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE + LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION + OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/patches/remove-rubygems-bundler.patch b/patches/remove-rubygems-bundler.patch new file mode 100644 index 0000000..831561a --- /dev/null +++ b/patches/remove-rubygems-bundler.patch @@ -0,0 +1,17 @@ +Description: we don't need bundler or rubygems +Author: Praveen Arimbrathodiyil +Last-Updated: 2013-05-17 + +--- a/spec/spec_helper.rb ++++ b/spec/spec_helper.rb +@@ -1,8 +1,8 @@ + $LOAD_PATH.unshift(File.dirname(__FILE__)) + $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), "..", "lib")) + +-require "bundler" +-Bundler.setup ++#require "bundler" ++#Bundler.setup + require "typhoeus" + require "rspec" + diff --git a/patches/series b/patches/series new file mode 100644 index 0000000..005fb51 --- /dev/null +++ b/patches/series @@ -0,0 +1 @@ +remove-rubygems-bundler.patch diff --git a/ruby-tests.rake b/ruby-tests.rake new file mode 100644 index 0000000..89a753d --- /dev/null +++ b/ruby-tests.rake @@ -0,0 +1,7 @@ +require 'rspec/core/rake_task' + +RSpec::Core::RakeTask.new(:spec) do |spec| + spec.pattern = './spec/**/*_spec.rb' +end + +task :default => :spec diff --git a/ruby-typhoeus.docs b/ruby-typhoeus.docs new file mode 100644 index 0000000..ceb1ea9 --- /dev/null +++ b/ruby-typhoeus.docs @@ -0,0 +1,2 @@ +CONTRIBUTING.md +README.md diff --git a/rules b/rules new file mode 100755 index 0000000..8f647e1 --- /dev/null +++ b/rules @@ -0,0 +1,20 @@ +#!/usr/bin/make -f +#export DH_VERBOSE=1 +# +# Uncomment to ignore all test failures (but the tests will run anyway) +#export DH_RUBY_IGNORE_TESTS=all +# +# Uncomment to ignore some test failures (but the tests will run anyway). +# Valid values: +#export DH_RUBY_IGNORE_TESTS=ruby1.8 ruby1.9.1 require-rubygems +# +# If you need to specify the .gemspec (eg there is more than one) +#export DH_RUBY_GEMSPEC=gem.gemspec +export GEM2DEB_TEST_RUNNER = --check-dependencies +export DH_RUBY = --gem-install + +%: + dh $@ --buildsystem=ruby --with ruby + +override_dh_installchangelogs: + dh_installchangelogs CHANGELOG.md diff --git a/salsa-ci.yml b/salsa-ci.yml new file mode 100644 index 0000000..33c3a64 --- /dev/null +++ b/salsa-ci.yml @@ -0,0 +1,4 @@ +--- +include: + - https://salsa.debian.org/salsa-ci-team/pipeline/raw/master/salsa-ci.yml + - https://salsa.debian.org/salsa-ci-team/pipeline/raw/master/pipeline-jobs.yml diff --git a/source/format b/source/format new file mode 100644 index 0000000..163aaf8 --- /dev/null +++ b/source/format @@ -0,0 +1 @@ +3.0 (quilt) diff --git a/upstream/metadata b/upstream/metadata new file mode 100644 index 0000000..2be6d08 --- /dev/null +++ b/upstream/metadata @@ -0,0 +1,4 @@ +Bug-Database: https://github.com/typhoeus/typhoeus/issues +Bug-Submit: https://github.com/typhoeus/typhoeus/issues/new +Repository: https://github.com/typhoeus/typhoeus.git +Repository-Browse: https://github.com/typhoeus/typhoeus/ diff --git a/watch b/watch new file mode 100644 index 0000000..a66b951 --- /dev/null +++ b/watch @@ -0,0 +1,2 @@ +version=4 +https://gemwatch.debian.net/typhoeus .*/typhoeus-(.*).tar.gz -- cgit v1.2.3 From 127d28369dfe544077939515518eed4cd1a40e81 Mon Sep 17 00:00:00 2001 From: Praveen Arimbrathodiyil Date: Thu, 18 Nov 2021 00:28:31 +0100 Subject: we don't need bundler or rubygems Last-Updated: 2013-05-17 Gbp-Pq: Name remove-rubygems-bundler.patch --- spec/spec_helper.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 73302cc..cdd58a8 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -1,8 +1,8 @@ $LOAD_PATH.unshift(File.dirname(__FILE__)) $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), "..", "lib")) -require "bundler" -Bundler.setup +#require "bundler" +#Bundler.setup require "typhoeus" require "rspec" -- cgit v1.2.3