summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--Changes3
-rw-r--r--META.json9
-rw-r--r--META.yml5
-rw-r--r--Makefile.PL1
-rw-r--r--debian/changelog20
-rw-r--r--debian/control7
-rw-r--r--debian/copyright2
-rw-r--r--debian/upstream/metadata6
-rw-r--r--lib/Finance/QuoteHist.pm7
-rw-r--r--lib/Finance/QuoteHist/Generic.pm9
-rw-r--r--lib/Finance/QuoteHist/Yahoo.pm144
-rw-r--r--t/dat/split_plain.dat2
-rw-r--r--t/dat/split_yahoo.dat2
13 files changed, 174 insertions, 43 deletions
diff --git a/Changes b/Changes
index 9968975..a5b3aca 100644
--- a/Changes
+++ b/Changes
@@ -1,5 +1,8 @@
Revision history for Perl extension Finance::QuoteHist.
+1.30 Fri 19 Feb 2021 01:47:26 PM EST
+ - Updated yahoo module to use json API
+
1.29 Wed Nov 13 10:27:10 EST 2019
- Fixed (harmless) dependency on google module
diff --git a/META.json b/META.json
index cc985d1..e6daf85 100644
--- a/META.json
+++ b/META.json
@@ -4,13 +4,13 @@
"unknown"
],
"dynamic_config" : 1,
- "generated_by" : "ExtUtils::MakeMaker version 7.24, CPAN::Meta::Converter version 2.150010",
+ "generated_by" : "ExtUtils::MakeMaker version 7.34, CPAN::Meta::Converter version 2.150010",
"license" : [
"unknown"
],
"meta-spec" : {
"url" : "http://search.cpan.org/perldoc?CPAN::Meta::Spec",
- "version" : "2"
+ "version" : 2
},
"name" : "Finance-QuoteHist",
"no_index" : {
@@ -35,6 +35,7 @@
"Date::Manip" : "0",
"HTML::TableExtract" : "2.07",
"HTTP::Request" : "0",
+ "JSON" : "0",
"LWP::UserAgent" : "0",
"MIME::Base64" : "0",
"Text::CSV" : "0"
@@ -42,6 +43,6 @@
}
},
"release_status" : "stable",
- "version" : "1.29",
- "x_serialization_backend" : "JSON::PP version 2.27400_02"
+ "version" : "1.30",
+ "x_serialization_backend" : "JSON::PP version 4.02"
}
diff --git a/META.yml b/META.yml
index 785bc6d..c52a788 100644
--- a/META.yml
+++ b/META.yml
@@ -7,7 +7,7 @@ build_requires:
configure_requires:
ExtUtils::MakeMaker: '0'
dynamic_config: 1
-generated_by: 'ExtUtils::MakeMaker version 7.24, CPAN::Meta::Converter version 2.150010'
+generated_by: 'ExtUtils::MakeMaker version 7.34, CPAN::Meta::Converter version 2.150010'
license: unknown
meta-spec:
url: http://module-build.sourceforge.net/META-spec-v1.4.html
@@ -21,8 +21,9 @@ requires:
Date::Manip: '0'
HTML::TableExtract: '2.07'
HTTP::Request: '0'
+ JSON: '0'
LWP::UserAgent: '0'
MIME::Base64: '0'
Text::CSV: '0'
-version: '1.29'
+version: '1.30'
x_serialization_backend: 'CPAN::Meta::YAML version 0.018'
diff --git a/Makefile.PL b/Makefile.PL
index ca58054..c6525c2 100644
--- a/Makefile.PL
+++ b/Makefile.PL
@@ -7,6 +7,7 @@ my %prereq_pm = (
'HTML::TableExtract' => 2.07,
'MIME::Base64' => 0,
'Text::CSV' => 0,
+ 'JSON' => 0,
);
eval "use Text::CSV_XS";
diff --git a/debian/changelog b/debian/changelog
index 30d41b9..c996b5f 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,23 @@
+libfinance-quotehist-perl (1.30-1) unstable; urgency=medium
+
+ * Team upload.
+
+ [ Debian Janitor ]
+ * Set upstream metadata fields: Bug-Database, Bug-Submit, Repository,
+ Repository-Browse.
+ * Update standards version to 4.5.0, no changes needed.
+
+ [ gregor herrmann ]
+ * Import upstream version 1.30.
+ - Updated yahoo module to use json API.
+ * Update years of upstream copyright.
+ * Add libjson-perl to test and runtime dependencies.
+ * Declare compliance with Debian Policy 4.5.1.
+ * Set Rules-Requires-Root: no.
+ * Bump debhelper-compat to 13.
+
+ -- gregor herrmann <gregoa@debian.org> Tue, 23 Feb 2021 22:01:04 +0100
+
libfinance-quotehist-perl (1.29-1) unstable; urgency=medium
* Team upload.
diff --git a/debian/control b/debian/control
index 7a03c69..181f453 100644
--- a/debian/control
+++ b/debian/control
@@ -4,18 +4,20 @@ Uploaders: Damyan Ivanov <dmn@debian.org>
Section: perl
Testsuite: autopkgtest-pkg-perl
Priority: optional
-Build-Depends: debhelper-compat (= 12)
+Build-Depends: debhelper-compat (= 13)
Build-Depends-Indep: libdate-manip-perl <!nocheck>,
libhtml-tableextract-perl <!nocheck>,
+ libjson-perl <!nocheck>,
libregexp-common-perl <!nocheck>,
libtext-csv-perl <!nocheck>,
libtext-csv-xs-perl <!nocheck>,
libwww-perl <!nocheck>,
perl
-Standards-Version: 4.4.1
+Standards-Version: 4.5.1
Vcs-Browser: https://salsa.debian.org/perl-team/modules/packages/libfinance-quotehist-perl
Vcs-Git: https://salsa.debian.org/perl-team/modules/packages/libfinance-quotehist-perl.git
Homepage: https://metacpan.org/release/Finance-QuoteHist
+Rules-Requires-Root: no
Package: libfinance-quotehist-perl
Architecture: all
@@ -23,6 +25,7 @@ Depends: ${misc:Depends},
${perl:Depends},
libdate-manip-perl,
libhtml-tableextract-perl,
+ libjson-perl,
libregexp-common-perl,
libtext-csv-xs-perl | libtext-csv-perl,
libwww-perl
diff --git a/debian/copyright b/debian/copyright
index 8cdb9b5..61a3269 100644
--- a/debian/copyright
+++ b/debian/copyright
@@ -4,7 +4,7 @@ Upstream-Contact: Matt P. Sisk <sisk@mojotoad.com>
Source: https://metacpan.org/release/Finance-QuoteHist
Files: *
-Copyright: 2000-2017, Matthew P. Sisk <sisk@mojotoad.com>
+Copyright: 2000-2021, Matthew P. Sisk <sisk@mojotoad.com>
License: Artistic or GPL-1+
Files: debian/*
diff --git a/debian/upstream/metadata b/debian/upstream/metadata
index a857c45..3545c73 100644
--- a/debian/upstream/metadata
+++ b/debian/upstream/metadata
@@ -1,2 +1,6 @@
+---
Archive: CPAN
-Repository: https://github.com/mojotoad/Finance-QuoteHist
+Bug-Database: https://github.com/mojotoad/Finance-QuoteHist/issues
+Bug-Submit: https://github.com/mojotoad/Finance-QuoteHist/issues/new
+Repository: https://github.com/mojotoad/Finance-QuoteHist.git
+Repository-Browse: https://github.com/mojotoad/Finance-QuoteHist
diff --git a/lib/Finance/QuoteHist.pm b/lib/Finance/QuoteHist.pm
index 622ab37..7bcb40a 100644
--- a/lib/Finance/QuoteHist.pm
+++ b/lib/Finance/QuoteHist.pm
@@ -10,7 +10,7 @@ use Carp;
use Finance::QuoteHist::Generic;
-$VERSION = '1.29';
+$VERSION = '1.30';
my @DEFAULT_ENGINES = qw(
Finance::QuoteHist::Yahoo
@@ -163,13 +163,12 @@ Matthew P. Sisk, E<lt>F<sisk@mojotoad.com>E<gt>
=head1 COPYRIGHT
-Copyright (c) 2000-2017 Matthew P. Sisk. All rights reserved. All wrongs
+Copyright (c) 2000-2021 Matthew P. Sisk. All rights reserved. All wrongs
revenged. This program is free software; you can redistribute it and/or
modify it under the same terms as Perl itself.
=head1 SEE ALSO
-Finance::QuoteHist::Generic(3), Finance::QuoteHist::Yahoo(3),
-Finance::QuoteHist::QuoteMedia(3), perl(1).
+Finance::QuoteHist::Generic(3), Finance::QuoteHist::Yahoo(3), perl(1).
=cut
diff --git a/lib/Finance/QuoteHist/Generic.pm b/lib/Finance/QuoteHist/Generic.pm
index ba6ee86..3a2d1f0 100644
--- a/lib/Finance/QuoteHist/Generic.pm
+++ b/lib/Finance/QuoteHist/Generic.pm
@@ -672,7 +672,12 @@ sub date_normalize {
$normal_date = ParseDate($m =~ /^\d+$/ ? "$y/$m/01" : "$m 01 $y");
}
else {
- $normal_date = ParseDate($date);
+ if ($date =~ /^\d{10}$/) {
+ $normal_date = ParseDateString("epoch $date");
+ }
+ else {
+ $normal_date = ParseDate($date);
+ }
}
$normal_date or return undef;
return $normal_date if $self->target_mode eq 'intraday';
@@ -1594,7 +1599,7 @@ Matthew P. Sisk, E<lt>F<sisk@mojotoad.com>E<gt>
=head1 COPYRIGHT
-Copyright (c) 2000-2017 Matthew P. Sisk. All rights reserved. All wrongs
+Copyright (c) 2000-2021 Matthew P. Sisk. All rights reserved. All wrongs
revenged. This program is free software; you can redistribute it and/or
modify it under the same terms as Perl itself.
diff --git a/lib/Finance/QuoteHist/Yahoo.pm b/lib/Finance/QuoteHist/Yahoo.pm
index f00bd63..dc615a7 100644
--- a/lib/Finance/QuoteHist/Yahoo.pm
+++ b/lib/Finance/QuoteHist/Yahoo.pm
@@ -10,6 +10,25 @@ use Finance::QuoteHist::Generic;
@ISA = qw(Finance::QuoteHist::Generic);
use Date::Manip;
+use JSON;
+
+# curl 'https://query2.finance.yahoo.com/v8/finance/chart/TLSA?formatted=true&crumb=l92p7dftYe%2F&lang=en-US&region=US&includeAdjustedClose=true&interval=1d&period1=1455840000&period2=1613692800&events=div%7Csplit&useYfid=true&corsDomain=finance.yahoo.com' \
+# -H 'authority: query2.finance.yahoo.com' \
+# -H 'pragma: no-cache' \
+# -H 'cache-control: no-cache' \
+# -H 'user-agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.182 Safari/537.36' \
+# -H 'dnt: 1' \
+# -H 'accept: */*' \
+# -H 'origin: https://finance.yahoo.com' \
+# -H 'sec-fetch-site: same-site' \
+# -H 'sec-fetch-mode: cors' \
+# -H 'sec-fetch-dest: empty' \
+# -H 'referer: https://finance.yahoo.com/quote/TLSA/history?period1=1455840000&period2=1613692800&interval=1d&filter=history&frequency=1d&includeAdjustedClose=true' \
+# -H 'accept-language: en-US,en;q=0.9' \
+# -H 'cookie: B=a912p61g2vk8r&b=3&s=tt; GUC=AQEBAQFgMSJgOUIaCwP5; A1=d=AQABBBvRL2ACEAlRbREoQmTRqOBpeDBZhKQFEgEBAQEiMWA5YAAAAAAA_SMAAAcIG9EvYDBZhKQ&S=AQAAApHY37COo3qJCJvBwk9D-TA; A3=d=AQABBBvRL2ACEAlRbREoQmTRqOBpeDBZhKQFEgEBAQEiMWA5YAAAAAAA_SMAAAcIG9EvYDBZhKQ&S=AQAAApHY37COo3qJCJvBwk9D-TA; A1S=d=AQABBBvRL2ACEAlRbREoQmTRqOBpeDBZhKQFEgEBAQEiMWA5YAAAAAAA_SMAAAcIG9EvYDBZhKQ&S=AQAAApHY37COo3qJCJvBwk9D-TA&j=US; cmp=t=1613746462&j=0; PRF=t%3DTLSA%252BIBM%252B%255EYHQ' \
+# -H 'sec-gpc: 1' \
+# --compressed
+
# https://query1.finance.yahoo.com/v7/finance/download/IBM?period1=1495391410&period2=1498069810&interval=1d&events=history&crumb=bB6k340lPXt
# https://query1.finance.yahoo.com/v7/finance/download/IBM?period1=993096000&period2=1498017600&interval=1wk&events=history&crumb=bB6k340lPXt
@@ -26,7 +45,7 @@ sub new {
my $class = ref($that) || $that;
my %parms = @_;
- $parms{parse_mode} = 'csv';
+ $parms{parse_mode} = 'json';
$parms{ua_params} ||= {};
$parms{ua_params}{cookie_jar} ||= {};
@@ -39,10 +58,6 @@ sub new {
$ticker = $ticker->[0] if ref $ticker eq 'ARRAY';
my $html = $self->fetch("https://finance.yahoo.com/quote/$ticker/history");
- open(F, ">/tmp/hmm.html");
- print F $html;
- close(F);
-
# extract the cookie crumb
my %crumbs;
for my $c ($html =~ /"crumb"\s*:\s*"([^"]+)"/g) {
@@ -66,32 +81,18 @@ sub new {
sub granularities { qw( daily weekly monthly ) }
-sub labels {
- my $self = shift;
- my %parms = @_;
- my $target_mode = $parms{target_mode} || $self->target_mode;
- my @labels;
- if ($target_mode eq 'split') {
- @labels = qw( date stock );
- }
- else {
- @labels = $self->SUPER::labels(%parms);
- push(@labels, 'adj') if $target_mode eq 'quote';
- }
- @labels;
-}
-
sub url_maker {
my($self, %parms) = @_;
my $target_mode = $parms{target_mode} || $self->target_mode;
- my $parse_mode = $parms{parse_mode} || $self->parse_mode;
+ my $parse_mode = $parms{parse_mode} || $self->parse_mode;
# *always* block unknown target mode and parse mode combinations for
# cascade to work properly!
return undef unless $target_mode eq 'quote' ||
$target_mode eq 'split' ||
$target_mode eq 'dividend';
- return undef unless $parse_mode eq 'html' || $parse_mode eq 'csv';
+
+ $parse_mode = "json";
my $granularity = lc($parms{granularity} || $self->granularity);
my $grain = 'd';
@@ -105,8 +106,24 @@ sub url_maker {
($start_date, $end_date) = ($end_date, $start_date);
}
- my $host = "query1.finance.yahoo.com";
- my $base_url = "https://$host/v7/finance/download/$ticker?";
+ #my $host = "query1.finance.yahoo.com";
+ #my $base_url = "https://$host/v7/finance/download/$ticker?";
+
+ # https://query2.finance.yahoo.com/v8/finance/chart/TLSA?
+ # formatted=true
+ # crumb=l92p7dftYe%2F
+ # lang=en-US
+ # region=US
+ # includeAdjustedClose=true
+ # interval=1d
+ # period1=1455840000
+ # period2=1613692800
+ # events=div%7Csplit
+ # useYfid=true
+ # corsDomain=finance.yahoo.com
+
+ my $host = "query2.finance.yahoo.com";
+ my $base_url = "https://$host/v8/finance/chart/$ticker?";
my @base_parms;
if ($start_date) {
my($y, $m, $d) = $self->ymd($start_date);
@@ -131,6 +148,7 @@ sub url_maker {
if ($target_mode eq "quote") {
push(@base_parms, "events=history");
+ push(@base_parms, "includeAdjustedClose=true")
}
elsif ($target_mode eq "dividend") {
push(@base_parms, "events=div");
@@ -145,6 +163,82 @@ sub url_maker {
return sub { pop @urls };
}
+sub json_parser {
+ my $self = shift;
+ my $target_mode = $self->target_mode();
+ my $json_quote_parse = sub {
+ my $data_result = shift;
+ my $data_indicators = $data_result->{indicators} || {};
+ my $data_quote = ($data_indicators->{quote} || [])->[0];
+ my @rows = [];
+ if ($data_quote) {
+ my $data_high = $data_quote->{high};
+ my $data_close = $data_quote->{close};
+ my $data_open = $data_quote->{open};
+ my $data_low = $data_quote->{low};
+ my $data_volume = $data_quote->{volume};
+ my $data_adj_close = $data_indicators->{adjclose}[0]{adjclose};
+ my $data_timestamp = $data_result->{timestamp};
+ for my $i (0 .. $#{$data_timestamp}) {
+ push(@rows, [
+ $data_timestamp->[$i],
+ $data_open->[$i],
+ $data_high->[$i],
+ $data_low->[$i],
+ $data_close->[$i],
+ $data_volume->[$i],
+ ]);
+ }
+ }
+ \@rows;
+ };
+ my $json_split_parse = sub {
+ my $data_result = shift;
+ my $data_events = $data_result->{events} || {};
+ my $data_splits = $data_events->{splits} || {};
+ my @rows;
+ if ($data_splits) {
+ for my $rec (sort values %$data_splits) {
+ push(@rows, [
+ $rec->{date},
+ $rec->{numerator},
+ $rec->{denominator},
+ ]);
+ }
+ }
+ \@rows;
+ };
+ my $json_div_parse = sub {
+ # "date" "amount"
+ my $data_result = shift;
+ my $data_events = $data_result->{events} || {};
+ my $data_dividends = $data_events->{dividends} || {};
+ my @rows;
+ for my $rec (sort values %$data_dividends) {
+ push(@rows, [
+ $rec->{date},
+ $rec->{amount},
+ ]);
+ }
+ \@rows;
+ };
+ sub {
+ my $data = shift;
+ $data = decode_json($data);
+ my $data_result = $data->{chart}{result}[0] || {};
+ if ($target_mode eq "quote") {
+ return $json_quote_parse->($data_result);
+ }
+ elsif ($target_mode eq "split") {
+ return $json_split_parse->($data_result);
+ }
+ elsif ($target_mode eq "dividend") {
+ return $json_div_parse->($data_result);
+ }
+ else { die "unknown mode: $target_mode" }
+ };
+}
+
1;
__END__
@@ -284,7 +378,7 @@ Matthew P. Sisk, E<lt>F<sisk@mojotoad.com>E<gt>
=head1 COPYRIGHT
-Copyright (c) 2000-2017 Matthew P. Sisk. All rights reserved. All wrongs
+Copyright (c) 2000-2021 Matthew P. Sisk. All rights reserved. All wrongs
revenged. This program is free software; you can redistribute it
and/or modify it under the same terms as Perl itself.
diff --git a/t/dat/split_plain.dat b/t/dat/split_plain.dat
index ba2047b..1eeca07 100644
--- a/t/dat/split_plain.dat
+++ b/t/dat/split_plain.dat
@@ -1,3 +1,3 @@
Finance::QuoteHist
NKE,2009/01/01,2013/10/01
-NKE:2012/12/26:1:2
+NKE:2012/12/26:2:1
diff --git a/t/dat/split_yahoo.dat b/t/dat/split_yahoo.dat
index 3bd9fb3..6ab170e 100644
--- a/t/dat/split_yahoo.dat
+++ b/t/dat/split_yahoo.dat
@@ -1,3 +1,3 @@
Finance::QuoteHist::Yahoo
NKE,2009/01/01,2013/10/01
-NKE:2012/12/26:1:2
+NKE:2012/12/26:2:1
generated by cgit v1.2.3 (git 2.25.1) at 2024-06-11 15:44:19 +0000