Hello community,
here is the log from the commit of package get-flash-videos for
openSUSE:Factory checked in at 2016-11-21 14:27:02
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Comparing /work/SRC/openSUSE:Factory/get-flash-videos (Old)
and /work/SRC/openSUSE:Factory/.get-flash-videos.new (New)
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "get-flash-videos"
Changes:
--------
--- /work/SRC/openSUSE:Factory/get-flash-videos/get-flash-videos.changes
2016-06-07 23:50:13.000000000 +0200
+++ /work/SRC/openSUSE:Factory/.get-flash-videos.new/get-flash-videos.changes
2016-11-21 14:27:03.000000000 +0100
@@ -1,0 +2,7 @@
+Mon Nov 21 00:55:54 UTC 2016 - jeng...@inai.de
+
+- Update to new upstream release 1.25.92
+ * Site updates, added basic HLSDownload that some sites have
+ started using.
+
+-------------------------------------------------------------------
Old:
----
1.25.91.tar.gz
New:
----
1.25.92.tar.gz
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Other differences:
------------------
++++++ get-flash-videos.spec ++++++
--- /var/tmp/diff_new_pack.uagwYw/_old 2016-11-21 14:27:04.000000000 +0100
+++ /var/tmp/diff_new_pack.uagwYw/_new 2016-11-21 14:27:04.000000000 +0100
@@ -1,7 +1,7 @@
#
# spec file for package get-flash-videos
#
-# Copyright (c) 2013-2015 SUSE LINUX Products GmbH, Nuernberg, Germany.
+# Copyright (c) 2016 SUSE LINUX GmbH, Nuernberg, Germany.
#
# All modifications and additions to the file contributed by third parties
# remain the property of their copyright owners, unless otherwise agreed
@@ -17,7 +17,7 @@
Name: get-flash-videos
-Version: 1.25.91
+Version: 1.25.92
Release: 0
Summary: A tool for downloading videos behind Flash loaders
License: Apache-2.0
@@ -37,10 +37,11 @@
Requires: perl(HTML::TokeParser)
Requires: perl(HTML::Tree)
Requires: perl(IPC::Open3)
-Requires: perl(List::Util)
Requires: perl(LWP::Protocol::https)
+Requires: perl(List::Util)
Requires: perl(MIME::Base64)
Requires: perl(Module::Find)
+Requires: perl(Term::ProgressBar)
Requires: perl(Time::HiRes)
Requires: perl(URI)
Requires: perl(URI::Escape)
++++++ 1.25.91.tar.gz -> 1.25.92.tar.gz ++++++
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/get-flash-videos-1.25.91/get_flash_videos
new/get-flash-videos-1.25.92/get_flash_videos
--- old/get-flash-videos-1.25.91/get_flash_videos 2015-11-19
22:24:22.000000000 +0100
+++ new/get-flash-videos-1.25.92/get_flash_videos 2016-05-11
17:03:49.000000000 +0200
@@ -42,6 +42,7 @@
use FlashVideo::Downloader;
use FlashVideo::RTMPDownloader;
use FlashVideo::FFmpegDownloader;
+use FlashVideo::HLSDownloader;
use FlashVideo::Search;
use FlashVideo::Utils;
use FlashVideo::VideoPreferences;
@@ -49,7 +50,7 @@
unshift @INC, \&plugin_loader;
# single line for MakeMaker to get version
-use constant CVERSION => "1.25.91"; our $VERSION = CVERSION;
+use constant CVERSION => "1.25.92"; our $VERSION = CVERSION;
our %opt;
BEGIN {
@@ -410,12 +411,13 @@
if(ref $data eq 'HASH') {
if (defined($data->{downloader}) && $data->{downloader} eq "ffmpeg") {
$downloader = FlashVideo::FFmpegDownloader->new;
- $file ||= $data->{flv};
+ } elsif (defined($data->{downloader}) && $data->{downloader} eq "hls") {
+ $downloader = FlashVideo::HLSDownloader->new;
} else {
# RTMP data
$downloader = FlashVideo::RTMPDownloader->new;
- $file ||= $data->{flv};
}
+ $file ||= $data->{flv};
} else {
# HTTP
$downloader = FlashVideo::Downloader->new;
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/get-flash-videos-1.25.91/lib/FlashVideo/HLSDownloader.pm
new/get-flash-videos-1.25.92/lib/FlashVideo/HLSDownloader.pm
--- old/get-flash-videos-1.25.91/lib/FlashVideo/HLSDownloader.pm
1970-01-01 01:00:00.000000000 +0100
+++ new/get-flash-videos-1.25.92/lib/FlashVideo/HLSDownloader.pm
2016-05-11 17:03:49.000000000 +0200
@@ -0,0 +1,87 @@
+# Part of get-flash-videos. See get_flash_videos for copyright.
+package FlashVideo::HLSDownloader;
+
+use strict;
+use warnings;
+use base 'FlashVideo::Downloader';
+use FlashVideo::Utils;
+use FlashVideo::FFmpegDownloader;
+use FlashVideo::JSON;
+use Term::ProgressBar;
+
+my $bitrate_index = {
+ high => 0,
+ medium => 1,
+ low => 2
+};
+
+sub download {
+ my ($self, $args, $file, $browser) = @_;
+
+ my $hls_url = $args->{args}->{hls_url};
+ my $prefs = $args->{args}->{prefs};
+
+ $browser->get($hls_url);
+ my %urls = read_hls_playlist($browser, $hls_url);
+
+ # Sort the urls and select the suitable one based upon quality preference
+ my $quality = $bitrate_index->{$prefs->{quality}};
+ my $min = $quality < scalar(keys(%urls)) ? $quality : scalar(keys(%urls));
+ my $key = (sort {int($b) <=> int($a)} keys %urls)[$min];
+
+ my ($hls_base, $trail) = ($hls_url =~ m/(.*\/)(.*)\.m3u8/);
+ my $filename_mp4 = $args->{flv};
+ my $filename_ts = $args->{flv} . ".ts";
+ my $video_url = $urls{$key} =~ m/http(s?):\/\// ? $urls{$key} :
$hls_base.$urls{$key};
+
+ $browser->get($video_url);
+
+ my @lines = split(/\r?\n/, $browser->content);
+ my @segments = ();
+
+ # Fill the url table
+ foreach my $line (@lines) {
+ if ($line !~ /#/) {
+ push @segments, $line;
+ }
+ }
+
+ my $i = 1;
+ my $num_segs = @segments;
+ info "Downloading segments";
+ my $progress_bar = Term::ProgressBar->new($num_segs);
+
+ open(my $fh_app, '>', $filename_ts) or die "Could not open file
$filename_ts";
+ binmode($fh_app);
+
+ foreach my $url (@segments) {
+ $browser->get($url);
+ print $fh_app $browser->content;
+ $progress_bar->update($i);
+ $i++;
+ }
+
+ # Use ffmpeg to clean up audio
+ my @ffmpeg_args = (
+ "-i", $filename_ts,
+ "-absf", "aac_adtstoasc",
+ "-c", "copy",
+ "-f", "mp4",
+ $filename_mp4
+ );
+
+ my $dl_args = {
+ downloader => "ffmpeg",
+ flv => $filename_mp4,
+ args => \@ffmpeg_args
+ };
+
+ my $ffmpeg_downloader = FlashVideo::FFmpegDownloader->new;
+ $ffmpeg_downloader->download($dl_args, $filename_mp4);
+
+ $self->{printable_filename} = $filename_mp4;
+
+ unlink $filename_ts;
+ return -s $filename_mp4;
+}
+1;
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/get-flash-videos-1.25.91/lib/FlashVideo/Mechanize.pm
new/get-flash-videos-1.25.92/lib/FlashVideo/Mechanize.pm
--- old/get-flash-videos-1.25.91/lib/FlashVideo/Mechanize.pm 2015-11-19
22:24:22.000000000 +0100
+++ new/get-flash-videos-1.25.92/lib/FlashVideo/Mechanize.pm 2016-05-11
17:03:49.000000000 +0200
@@ -72,7 +72,8 @@
if($App::get_flash_videos::opt{debug}) {
my $text = join " ", $self->response->code,
- $self->response->header("Content-type"), "(" . length($self->content) .
")";
+ $self->response->header("Content-type"),
$self->response->header("Content-length"),
+ "(" . length($self->content) . ")";
$text .= ": " . DBI::data_string_desc($self->content) if eval { require
DBI };
print STDERR "<- $text\n";
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/get-flash-videos-1.25.91/lib/FlashVideo/Site/Arte.pm
new/get-flash-videos-1.25.92/lib/FlashVideo/Site/Arte.pm
--- old/get-flash-videos-1.25.91/lib/FlashVideo/Site/Arte.pm 2015-11-19
22:24:22.000000000 +0100
+++ new/get-flash-videos-1.25.92/lib/FlashVideo/Site/Arte.pm 2016-05-11
17:03:49.000000000 +0200
@@ -10,19 +10,10 @@
sub find_video {
my ($self, $browser, $embed_url, $prefs) = @_;
- my ($lang, $jsonurl, $filename, $title, $videourl, $quality);
+ my ($jsonurl, $filename, $title, $videourl, $quality);
debug "Arte::find_video called, embed_url = \"$embed_url\"\n";
- my $pageurl = $browser->uri() . "";
- if($pageurl =~ /www\.arte\.tv\/guide\/(..)\//) {
- $lang = $1;
- } elsif($pageurl =~ /concert.arte.tv\/(..)\//) {
- $lang = $1;
- } else {
- die "Unable to find language in original URL \"$pageurl\"\n";
- }
-
if($browser->content =~ /arte_vp_url=['"](.*)['"]/) {
$jsonurl = $1;
debug "found arte_vp_url \"$jsonurl\"\n";
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/get-flash-videos-1.25.91/lib/FlashVideo/Site/Dplay.pm
new/get-flash-videos-1.25.92/lib/FlashVideo/Site/Dplay.pm
--- old/get-flash-videos-1.25.91/lib/FlashVideo/Site/Dplay.pm 1970-01-01
01:00:00.000000000 +0100
+++ new/get-flash-videos-1.25.92/lib/FlashVideo/Site/Dplay.pm 2016-05-11
17:03:49.000000000 +0200
@@ -0,0 +1,42 @@
+# Part of get-flash-videos. See get_flash_videos for copyright.
+package FlashVideo::Site::Dplay;
+
+use strict;
+use FlashVideo::Utils;
+use FlashVideo::JSON;
+use HTTP::Cookies;
+use URI::Escape;
+
+our $VERSION = '0.01';
+sub Version() { $VERSION;}
+
+sub find_video {
+ my ($self, $browser, $embed_url, $prefs) = @_;
+ my $title = extract_title($browser);
+ my $video_id = ($browser->content =~ /data-video-id="([0-9]*)"/)[0];
+ my $url =
"https://secure.dplay.se/secure/api/v2/user/authorization/stream/$video_id?stream_type=hls";;
+
+ my $cookies = HTTP::Cookies->new;
+ $cookies->set_cookie(0, 'dsc-geo', uri_escape('{"countryCode": "SE"}'), '/',
'secure.dplay.se');
+ $browser->cookie_jar($cookies);
+ $browser->get($url);
+
+ my $filename = title_to_filename($title, "mp4");
+
+ my $jsonstr = $browser->content;
+ my $json = from_json($jsonstr);
+ my $hls_url = $json->{hls};
+
+ if ($json->{type} eq "drm") {
+ die "Does not support DRM videos";
+ }
+
+ return {
+ downloader => "hls",
+ flv => $filename,
+ args => { hls_url => $hls_url, prefs => $prefs}
+ };
+
+}
+
+1;
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/get-flash-videos-1.25.91/lib/FlashVideo/Site/Itv.pm
new/get-flash-videos-1.25.92/lib/FlashVideo/Site/Itv.pm
--- old/get-flash-videos-1.25.91/lib/FlashVideo/Site/Itv.pm 2015-11-19
22:24:22.000000000 +0100
+++ new/get-flash-videos-1.25.92/lib/FlashVideo/Site/Itv.pm 2016-05-11
17:03:49.000000000 +0200
@@ -5,8 +5,9 @@
use FlashVideo::Utils;
use HTML::Entities;
use Encode;
+use Data::Dumper;
-our $VERSION = '0.07';
+our $VERSION = '0.08';
sub Version() { $VERSION;}
sub find_video {
@@ -219,14 +220,78 @@
}
}
- return {
+ my $dlparams = {
rtmp => $rtmp,
playpath => $playpath,
flv => $flv,
- swfhash($browser, "http://www.itv.com/mercury/Mercury_VideoPlayer.swf";)
+ itv_swfhash($browser, "http://www.itv.com/mercury/Mercury_VideoPlayer.swf";)
};
+
+ if ($dlparams->{swfsize} < 10) {
+ # Use hardcoded value if failed
+ print STDERR "Size too small - using hardcoded values for swf\n";
+ $dlparams->{swfUrl} = 'http://www.itv.com/mercury/Mercury_VideoPlayer.swf';
+ $dlparams->{swfsize} = 990750;
+ $dlparams->{swfhash} =
'b6c8966da3f49610be7178b01ca33d046bbf915e2908d9dafe11e4b042d8eeea';
+ }
+ return $dlparams;
+}
+
+
+use constant FP_KEY => "Genuine Adobe Flash Player 001";
+
+# Replacement swfhash upto version 19
+sub itv_swfhash {
+ my($browser, $url) = @_;
+
+ $browser->get($url);
+
+ return itv_swfhash_data($browser->content, $url);
}
+sub itv_swfhash_data {
+ my ($data, $url) = @_;
+
+ die "Must have Digest::SHA for this RTMP download\n"
+ unless eval {
+ require Digest::SHA;
+ };
+
+ # swf file header
+ # swf signature type FWS uncompressed, CWS Zlib compression, ZWS LZMA
compression
+ # swf version
+ my ($swftype, $swfversion , $swfsize) = unpack ("a3CI", substr($data, 0, 8));
+
+ print STDERR "swf type = $swftype version = $swfversion size = $swfsize\n";
+
+ if ($swftype eq 'CWS' ) {
+
+ die "Must have Compress::Zlib for this RTMP download\n"
+ unless eval {
+ require Compress::Zlib;
+ };
+
+ # sfw uncompressed header.
+ $data = "F" . substr($data, 1, 7)
+ . Compress::Zlib::uncompress(substr $data, 8);
+
+ } elsif ($swftype eq 'ZWS') {
+ # swf version 13 and later
+ print STDERR "Warning Lzma not supported\n";
+ } elsif ($swftype ne 'FWS') {
+ print STDERR "Warning Not a SWF Format file\n";
+ }
+
+ my $datalen = length $data;
+ if ($datalen != $swfsize) {
+ print STDERR "swf size $swfsize doesn't match uncompressed size
$datalen\n";
+ }
+
+ return
+ swfsize => $datalen,
+ swfhash => Digest::SHA::hmac_sha256_hex($data, FP_KEY),
+ swfUrl => $url;
+}
1;
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/get-flash-videos-1.25.91/lib/FlashVideo/Site/Kanal5play.pm
new/get-flash-videos-1.25.92/lib/FlashVideo/Site/Kanal5play.pm
--- old/get-flash-videos-1.25.91/lib/FlashVideo/Site/Kanal5play.pm
2015-11-19 22:24:22.000000000 +0100
+++ new/get-flash-videos-1.25.92/lib/FlashVideo/Site/Kanal5play.pm
1970-01-01 01:00:00.000000000 +0100
@@ -1,122 +0,0 @@
-# Part of get-flash-videos. See get_flash_videos for copyright.
-package FlashVideo::Site::Kanal5play;
-
-use strict;
-use warnings;
-use FlashVideo::Utils;
-use FlashVideo::JSON;
-
-our $VERSION = '0.05';
-sub Version() { $VERSION;}
-
-my $bitrate_index = {
- high => 0,
- medium => 1,
- low => 2
-};
-
-sub find_video {
- my ($self, $browser, $embed_url, $prefs) = @_;
- if (!($browser->uri->as_string =~ m/video\/([0-9]*)/)) {
- die "No video id found in url";
- }
- my $video_id = $1;
- my $info_url =
"http://www.kanal5play.se/api/getVideo?format=IPAD&videoId=$video_id";;
-
- $browser->get($info_url);
-
- if (!$browser->success) {
- die "Couldn't download $info_url: " . $browser->response->status_line;
- }
-
- my $jsonstr = $browser->content;
- my $json = from_json($jsonstr);
- my $name = $json->{program}->{name};
- my $episode = $json->{episodeNumber};
- my $season = $json->{seasonNumber};
- my $subtitle = $json->{hasSubtitle};
- my $title = sprintf "%s - S%02dE%02d", $name, $season, $episode;
- my $hls_m3u = $json->{streams}[0]->{source};
-
- my %paths = read_hls_playlist($browser, $hls_m3u);
-
- # Sort the paths and select the suitable one based upon quality preference
- my $quality = $bitrate_index->{$prefs->{quality}};
- my $min = $quality < scalar(keys(%paths)) ? $quality : scalar(keys(%paths));
- my $key = (sort {int($b) <=> int($a)} keys %paths)[$min];
-
- my $video_url = $paths{$key};
-
- my $hls_base = $hls_m3u;
- $hls_base =~ s/playlist\.m3u8//;
-
- my $filename = title_to_filename($title, "mp4");
-
- # Set the arguments for ffmpeg
- my @ffmpeg_args = (
- "-i", "$hls_base$video_url",
- "-acodec", "copy",
- "-vcodec", "copy",
- "-absf", "aac_adtstoasc",
- "-f", "mp4",
- "$filename"
- );
-
- # Check for subtitles
- if ($prefs->{subtitles} and $subtitle) {
- my $subtitle_url = "http://www.kanal5play.se/api/subtitles/$video_id";;
- $browser->get($subtitle_url);
- if (!$browser->success) {
- die "Couldn't download $subtitle_url: " .
$browser->response->status_line;
- }
- $jsonstr = $browser->content;
- $json = from_json($jsonstr);
-
- # The format is a list of hashmap with the following keys:
- # startMillis : int
- # endMillis : int
- # text : string
- # posX : int
- # posY : int
- # colorR : int
- # colorG : int
- # colorB : int
- #
- # We convert this to an srt
-
- my $srt_filename = title_to_filename($title, "srt");
- open my $srt_fh, '>', $srt_filename
- or die "Can't open subtitles file $srt_filename: $!";
-
- my $i = 1;
-
- foreach my $line (@{$json}) {
- my $text = $line->{text};
- my $hour = int($line->{startMillis}) / 3600000;
- my $min = (int($line->{startMillis}) / 60000) % 60;
- my $sec = (int($line->{startMillis}) / 1000) % 60;
- my $milli = int($line->{startMillis}) % 1000;
-
- my $start = sprintf "%02d:%02d:%02d,%03d", $hour, $min, $sec, $milli;
-
- $hour = int($line->{endMillis}) / 3600000;
- $min = (int($line->{endMillis}) / 60000) % 60;
- $sec = (int($line->{endMillis}) / 1000) % 60;
- $milli = int($line->{endMillis}) % 1000;
-
- my $end = sprintf "%02d:%02d:%02d,%03d", $hour, $min, $sec, $milli;
-
- print $srt_fh "$i\n" . "$start --> $end\n" . "$text\n\n";
-
- $i++;
- }
- close $srt_fh;
- }
-
- return {
- downloader => "ffmpeg",
- flv => $filename,
- args => \@ffmpeg_args
- };
-}
-1;
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/get-flash-videos-1.25.91/lib/FlashVideo/Site/Pbs.pm
new/get-flash-videos-1.25.92/lib/FlashVideo/Site/Pbs.pm
--- old/get-flash-videos-1.25.91/lib/FlashVideo/Site/Pbs.pm 2015-11-19
22:24:22.000000000 +0100
+++ new/get-flash-videos-1.25.92/lib/FlashVideo/Site/Pbs.pm 2016-05-11
17:03:49.000000000 +0200
@@ -3,19 +3,17 @@
use strict;
use warnings;
-use Encode;
use FlashVideo::Utils;
-use MIME::Base64 qw(decode_base64);
+use FlashVideo::JSON;
=pod
Programs that work:
- http://video.pbs.org/video/1623753774/
- -
http://www.pbs.org/wnet/nature/episodes/revealing-the-leopard/full-episode/6084/
+ - http://www.pbs.org/video/2365612568/
- http://www.pbs.org/wgbh/nova/ancient/secrets-stonehenge.html
- -
http://www.pbs.org/wnet/americanmasters/episodes/lennonyc/outtakes-jack-douglas/1718/
+ - http://www.pbs.org/show/bletchley-circle/
-
http://www.pbs.org/wnet/need-to-know/video/need-to-know-november-19-2010/5189/
- -
http://www.pbs.org/newshour/bb/transportation/july-dec10/airport_11-22.html
Programs that don't work yet:
- http://www.pbs.org/wgbh/pages/frontline/woundedplatoon/view/
@@ -26,38 +24,48 @@
=cut
-our $VERSION = '0.02';
+our $VERSION = '0.03';
sub Version() { $VERSION; }
sub find_video {
+ our %opt;
my ($self, $browser, $embed_url, $prefs) = @_;
- die "Must have Crypt::Rijndael installed to download from PBS"
- unless eval { require Crypt::Rijndael };
-
- my ($media_id) = $embed_url =~
m[http://video\.pbs\.org/videoPlayerInfo/(\d+)]x;
+ my ($media_id) = $embed_url =~
m[http://(?:video|www)\.pbs\.org/videoPlayerInfo/(\d+)]x;
+ debug("media id found in URL") if (defined $media_id);
unless (defined $media_id) {
+ debug("media id not found in URL");
($media_id) = $browser->uri->as_string =~ m[
- ^http://video\.pbs\.org/video/(\d+)
+ ^http://(?:video|www)\.pbs\.org/video/(\d+)
]x;
+ debug("media id found in URI") if (defined $media_id);
}
unless (defined $media_id) {
+ debug("media id not found in URI");
($media_id) = $browser->content =~ m[
- http://video\.pbs\.org/widget/partnerplayer/(\d+)
+ http://(?:video|www)\.pbs\.org/widget/partnerplayer/(\d+)
]x;
+ debug("media id found in partner player link") if (defined $media_id);
}
unless (defined $media_id) {
+ debug("media id not found in partner player link");
($media_id) = $browser->content =~ m[
/embed-player[^"]+\bepisodemediaid=(\d+)
]x;
+ debug("media id found in embedded player reference") if (defined
$media_id);
}
unless (defined $media_id) {
+ debug("media id not found in embedded player reference");
($media_id) = $browser->content =~ m[var videoUrl = "([^"]+)"];
+ debug("media id found in a pbs_video_id tag") if (defined $media_id);
}
unless (defined $media_id) {
+ debug("media id not found in a javascript videoURL variable");
($media_id) = $browser->content =~ m[pbs_video_id_\S+" value="([^"]+)"];
+ debug("media id found in a pbs_video_id tag") if (defined $media_id);
}
unless (defined $media_id) {
+ debug("media id not found in a pbs_video_id tag");
my ($pap_id, $youtube_id) = $browser->content =~ m[
\bDetectFlashDecision\ \('([^']+)',\ '([^']+)'\);
]x;
@@ -67,75 +75,189 @@
require FlashVideo::Site::Youtube;
return FlashVideo::Site::Youtube->find_video($browser, $url, $prefs);
}
+ debug("media id not found in a YouTube tag");
}
- die "Couldn't find media_id\n" unless defined $media_id;
- debug "media_id: $media_id\n";
+ # pbs.org uses redirects all over the place
$browser->allow_redirects;
- $browser->get("http://video.pbs.org/videoPlayerInfo/$media_id";);
- debug "fetched: $media_id\n";
+
+ if (! defined $media_id) {
+ debug ("...scanning for list of multiple videos");
- my $xml = $browser->content;
- debug "retrieved xml: $media_id\n";
+ my @possible_videos = $browser->content =~ m{<a
href=['"](/video/\d+/)['"][^>]*>([^<]+)</a>}g;
+ if (@possible_videos) {
+ if (!$opt{yes}) {
+ print "There are " . scalar(@possible_videos)/2 . " videos referenced,
please choose:\n";
+ my $count;
+ for (my $i = 0; $i < $#possible_videos; $i += 2) {
+ my $item = $i/2;
+ my $item_title = $possible_videos[$i+1];
+ print "$item - $item_title\n";
+ }
+
+ print "\nWhich video would you like to use?: ";
+ chomp(my $chosen_item = <STDIN>);
+ $chosen_item *= 2;
+ if ($possible_videos[$chosen_item]) {
+ my $chosen_url = "http://www.pbs.org"; .
$possible_videos[$chosen_item];
+ $browser->get($chosen_url);
+ return $self->find_video($browser, $chosen_url, $prefs);
+ }
+ }
+ else
+ {
+ info "There were " . scalar(@possible_videos)/2 . " referenced videos,
but you used the yes option.";
+ info "Re-run without the yes option to select one.";
+ }
+ }
+ }
+
+ #
+ die "Couldn't find media_id\n" unless defined $media_id;
+ debug "media_id: $media_id\n";
+
+ my $account = $prefs->account("pbs.org", <<EOT);
+If you set up a PBS account, you can access high definition videos.
+The pbs.org login is the email address you registered at pbs.org.
+See the documentation, i.e man netrc, for how to configure ~/.netrc
+and skip continual prompting for account credentials. Example:
+ machine pbs.org
+ login myemail\@xyzzy.net
+ password xxxxxxx
+NOTE: if the login is set to 'no', standard definition will be downloaded.
+
+EOT
+
+ my $query = 'http://player.pbs.org/portalplayer/' . $media_id;
+
+ if ($account->username and $account->username ne 'no' and
$account->password) {
+ # get the pbs.ord login page and fill in the login form
+
$browser->get('https://account.pbs.org/oauth2/authorize/?scope=account&redirect_uri=http://www.pbs.org/login/&response_type=code&client_id=LXLFIaXOVDsfS850bnvsxdcLKlvLStjRBoBWbFRE');
+ if (! $browser->success()) {
+ debug $browser->content();
+ die "Could not access login page" unless $browser->success();
+ }
+
+ # fill in the login form with the users credentials
+ $browser->form_number(1);
+ $browser->field('email', $account->username);
+ $browser->field('password', $account->password);
+
+ # submit the login request
+ $browser->submit();
+ if ($browser->success()) {
+
+ # login successful, but need to extract some cookie values to retrieve
+ # high definition video
+ my $pbs_uid;
+ my $pbs_station;
+
+ foreach my $cookie (split /\n/, $browser->cookie_jar->as_string()) {
+ my @tokens = split /; |: /, $cookie;
+ my ($cname, $cvalue) = split /=/, $tokens[1];
+ $pbs_uid = $cvalue if $cname eq 'pbs_uid';
+ $pbs_station = $cvalue if $cname eq 'pbsol.station';
+ debug "cookie name = $cname, value = $cvalue"
+ }
+
+ debug "setting pbs_uid=$pbs_uid and callsign=$pbs_station";
+ info "using pbs.org account " . $account->username . " to retrieve high
definition videos";
+ # format query to get high definition video details in JSON
+ $query = $query . '/?callsign=' . $pbs_station . '&uid=' . $pbs_uid .
'&callback=video_info&format=jsonp&type=portal';
+
+ } else {
+ info "\n*** pbs.org login failed ***\ncorrect your login and
password\nwill retrieve standard definition video.\n";
+ # format query to get standard definition video details in JSON
+ $query = $query .
'/?callsign=KCTS&callback=video_info&format=jsonp&type=portal';
+ }
+
+ } else {
+ info "no pbs login credentials, will retrieve standard definition video.";
+ # format query to get standard definition video details in JSON
+ $query = $query .
'/?callsign=KCTS&callback=video_info&format=jsonp&type=portal';
+ }
- $xml = encode('utf-8', $xml);
- debug "encode: $media_id\n";
+ info "Downloading video metadata";
+ $browser->get($query);
+ die "Could not get video metadata" unless $browser->success();
- #$xml =~ s/&/&/g; # not sure this is needed anymore
- #debug "decoded ampersands: $media_id\n";
+ # PBS returns the video metadata as a javascript variable
+ # extract the embedded javascript and extract the PBS.videoData variable
+ my @scriptags = $browser->content() =~/<script[^>]*>(.+?)<\/script>/sig;
+ my $script;
+ my $pbsdata;
+ local $/ = "\r\n";
+ foreach $script (@scriptags)
+ {
+ if ($script =~ /PBS.videoData/si) {
+ ($pbsdata) = $script =~ /PBS.videoData += +([^;]*);/s;
+ # change ' to " for the json parser
+ $pbsdata =~ s/'/"/g;
+ # PBS computes the number of chapters in the javascript.
+ # We don't care, so replace it with an integer.'
+ $pbsdata =~ s/: *chapters *,/: 4,/g;
+ debug $pbsdata;
+ last;
+ }
+ }
+# Parse the json structure
+ my $result = from_json($pbsdata);
+ debug Data::Dumper::Dumper($result);
- my $href = from_xml($xml);
- debug "from_xml: $media_id\n";
+ # Get the video's title and urs source
+ my $title = $result->{title};
+ die "Could not extract video title" unless $title;
+ debug "title is: $title\n";
- my $file = $href->{videoInfo}->{title};
- debug "title is: $file\n";
+ my $urs = $result->{recommended_encoding}->{url};
+ die "Could not extract video urs" unless $urs;
+ debug "urs extracted\n";
- my $release_url = $href->{releaseURL};
- debug "release_url is: $release_url\n";
-
- unless ($release_url =~ m[^https?://]) {
- debug "encrypted release url: $release_url\n";
- my ($type, $iv, $ciphertext) = split '\$', $release_url, 3;
- $release_url = undef;
-
- # From http://www-tc.pbs.org/video/media/swf/PBSPlayer.swf
- my $key = 'RPz~i4p*FQmx>t76';
-
- my $cipher = Crypt::Rijndael->new($key, Crypt::Rijndael->MODE_CBC);
- $iv = pack 'H*', $iv if 32 == length $iv;
- $cipher->set_iv($iv);
-
- $release_url = $cipher->decrypt(decode_base64($ciphertext));
- $release_url =~ s/\s+$//;
- }
- debug "unencrypted release url: $release_url\n";
-
- $browser->prohibit_redirects;
- $browser->get($release_url);
- debug "retrieved release_url: $release_url\n";
-
- my $rtmp_url = $browser->res->header('location')
- || from_xml($browser->content)->{choice}{url}
- || die "Couldn't find stream url\n";
- $rtmp_url =~ s/<break>//;
- debug "rtmp_url: $rtmp_url\n";
+ # format another query to get video url in JSON
+ $query = $urs . '?format=json';
- my $playpath;
- my $filetype;
- ($playpath, $filetype) = $rtmp_url =~ m[/(([^/:]*):videos.*$)];
- debug "playpath: $playpath\n";
- debug "file type: $filetype\n";
-
- if(!$file) {
- ($file) = $rtmp_url =~ m{([^/\?]+)$};
+ info "Downloading video details";
+ $browser->get($query);
+ die "Could not get video details" unless $browser->success();
+
+ # Content is JSON fomatted
+ $result = from_json($browser->content());
+
+ # Get the video's url source
+ my $url = $result->{url};
+ die "Could not extract video url. Possibly it is no longer available."
unless $url;
+ debug "found PBS video: $media_id @ $url";
+
+ # get the scheme and filetype to determine appropriate downloader
+ my ($scheme, $filetype) = $url =~ m[(^\w+):.*\.(\w+)$];
+ debug "scheme is: $scheme";
+ debug "file type is: $filetype";
+
+ if ($scheme =~ m[^rtmp]) {
+ # pbs.org has not moved all videos from flash to hls
+ # use rtmpdump for backward compatibility
+ my $playpath;
+ ($playpath) = $url =~ m[(\w+:*:videos.*$)];
+ debug "playpath is: $playpath";
+ debug "using rtmp downloader";
+ return {
+ rtmp => $url,
+ playpath => $playpath,
+ flashVer => 'LNX 11,2,202,481',
+ flv => title_to_filename($title, $filetype),
+ };
+ } elsif ($scheme =~ m[^http] and $filetype eq "m3u8") {
+ debug "using hls downloader";
+ return {
+ downloader => "hls",
+ flv => title_to_filename($title, "mp4"),
+ args => { hls_url => $url, prefs => $prefs }
+ };
+ } elsif ($scheme =~ m[^http] and $filetype eq "mp4") {
+ return $url, title_to_filename($title, $filetype);
+ } else {
+ die "Video is in unknown scheme or format. Run with debug and report
problem";
}
-
- return {
- rtmp => $rtmp_url,
- playpath => $playpath,
- flashVer => 'LNX 11,2,202,481',
- flv => title_to_filename($file, $filetype),
- };
}
1;
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/get-flash-videos-1.25.91/lib/FlashVideo/Site/Svtplay.pm
new/get-flash-videos-1.25.92/lib/FlashVideo/Site/Svtplay.pm
--- old/get-flash-videos-1.25.91/lib/FlashVideo/Site/Svtplay.pm 2015-11-19
22:24:22.000000000 +0100
+++ new/get-flash-videos-1.25.92/lib/FlashVideo/Site/Svtplay.pm 2016-05-11
17:03:49.000000000 +0200
@@ -6,17 +6,10 @@
use FlashVideo::Utils;
use FlashVideo::JSON;
-use HTML::Entities;
-our $VERSION = '0.04';
+our $VERSION = '0.06';
sub Version() { $VERSION;}
-my $bitrate_index = {
- high => 0,
- medium => 1,
- low => 2
-};
-
sub find_video_svt {
my ($self, $browser, $embed_url, $prefs, $oppet_arkiv) = @_;
my @rtmpdump_commands;
@@ -26,8 +19,7 @@
}
my $vid_type = $1;
my $video_id = $2;
- $browser->content =~ m/<title>(.+)<\/title>/;
- my $name = decode_entities($1);
+ my $name = extract_title($browser);
my $info_url = $oppet_arkiv ?
"http://www.oppetarkiv.se/video/$video_id?output=json"; :
"http://www.svtplay.se/$vid_type/$video_id?output=json"; ;
@@ -79,32 +71,13 @@
# If we found an m3u8 file we generate the ffmpeg download command
if (!($m3u8 eq "")) {
-
- my %urls = read_hls_playlist($browser, $m3u8);
-
- # Sort the urls and select the suitable one based upon quality preference
- my $quality = $bitrate_index->{$prefs->{quality}};
- my $min = $quality < scalar(keys(%urls)) ? $quality : scalar(keys(%urls));
- my $key = (sort {int($b) <=> int($a)} keys %urls)[$min];
-
- my $video_url = $urls{$key};
my $filename = title_to_filename($name, "mp4");
- # Set the arguments for ffmpeg
- my @ffmpeg_args = (
- "-i", "$video_url",
- "-acodec", "copy",
- "-vcodec", "copy",
- "-absf", "aac_adtstoasc",
- "-f", "mp4",
- "$filename"
- );
-
return {
- downloader => "ffmpeg",
+ downloader => "hls",
flv => $filename,
- args => \@ffmpeg_args
- };
+ args => { hls_url => $m3u8, prefs => $prefs }
+ }
} else {
return {
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/get-flash-videos-1.25.91/lib/FlashVideo/Site/Tv3play.pm
new/get-flash-videos-1.25.92/lib/FlashVideo/Site/Tv3play.pm
--- old/get-flash-videos-1.25.91/lib/FlashVideo/Site/Tv3play.pm 2015-11-19
22:24:22.000000000 +0100
+++ new/get-flash-videos-1.25.92/lib/FlashVideo/Site/Tv3play.pm 2016-05-11
17:03:49.000000000 +0200
@@ -4,7 +4,7 @@
use warnings;
use FlashVideo::Utils;
use FlashVideo::JSON;
-our $VERSION = '0.06';
+our $VERSION = '0.07';
sub Version() { $VERSION;}
sub find_video {
@@ -54,31 +54,12 @@
# Prefer hls stream since it contains better video format
if ($json->{streams}->{hls}) {
my $hls_url = $json->{streams}->{hls};
-
- my %urls = read_hls_playlist($browser, $hls_url);
-
- # Sort the urls and select the suitable one based upon quality preference
- my $quality = $bitrate_index->{$prefs->{quality}};
- my $min = $quality < scalar(keys(%urls)) ? $quality : scalar(keys(%urls));
- my $key = (sort {int($b) <=> int($a)} keys %urls)[$min];
-
- my ($hls_base, $trail) = ($hls_url =~ m/(.*\/)(.*)\.m3u8/);
my $filename = title_to_filename($title, "mp4");
- my $video_url = $urls{$key} =~ m/http:\/\// ? $urls{$key} :
$hls_base.$urls{$key};
-
- my @ffmpeg_args = (
- "-i", "$video_url",
- "-acodec", "copy",
- "-vcodec", "copy",
- "-absf", "aac_adtstoasc",
- "-f", "mp4",
- "$filename"
- );
return {
- downloader => "ffmpeg",
+ downloader => "hls",
flv => $filename,
- args => \@ffmpeg_args
+ args => { hls_url => $hls_url, prefs => $prefs }
};
}
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/get-flash-videos-1.25.91/lib/FlashVideo/Site/Tv4play.pm
new/get-flash-videos-1.25.92/lib/FlashVideo/Site/Tv4play.pm
--- old/get-flash-videos-1.25.91/lib/FlashVideo/Site/Tv4play.pm 2015-11-19
22:24:22.000000000 +0100
+++ new/get-flash-videos-1.25.92/lib/FlashVideo/Site/Tv4play.pm 2016-05-11
17:03:49.000000000 +0200
@@ -5,7 +5,7 @@
use FlashVideo::Utils;
use List::Util qw(reduce);
-our $VERSION = '0.02';
+our $VERSION = '0.03';
sub Version() { $VERSION;}
my $bitrate_index = {
@@ -17,7 +17,7 @@
sub find_video {
my ($self, $browser, $embed_url, $prefs) = @_;
my $video_id = ($embed_url =~ /video_id=([0-9]*)/)[0];
- my $smi_url =
"http://premium.tv4play.se/api/web/asset/$video_id/play?protocol=hls";;
+ my $smi_url =
"http://prima.tv4play.se/api/web/asset/$video_id/play?protocol=hls";;
my $title = extract_title($browser);
$browser->get($smi_url);
my $content = from_xml($browser);
@@ -70,32 +70,12 @@
}
}
- my %urls = read_hls_playlist($browser, $hls_m3u);
-
- # Sort the urls and select the suitable one based upon quality preference
- my $quality = $bitrate_index->{$prefs->{quality}};
- my $min = $quality < scalar(keys(%urls)) ? $quality : scalar(keys(%urls));
- my $key = (sort {int($b) <=> int($a)} keys %urls)[$min];
-
- my $video_url = $urls{$key};
my $filename = title_to_filename($title, "mp4");
- my $url = $video_url =~ m/http:\/\// ? $video_url : $hls_base.$video_url;
-
- # Set the arguments for ffmpeg
- my @ffmpeg_args = (
- "-i", "$url",
- "-acodec", "copy",
- "-vcodec", "copy",
- "-absf", "aac_adtstoasc",
- "-f", "mp4",
- "$filename"
- );
-
return {
- downloader => "ffmpeg",
+ downloader => "hls",
flv => $filename,
- args => \@ffmpeg_args
+ args => { hls_url => $hls_m3u, prefs => $prefs }
};
}
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/get-flash-videos-1.25.91/lib/FlashVideo/Site/Vimeo.pm
new/get-flash-videos-1.25.92/lib/FlashVideo/Site/Vimeo.pm
--- old/get-flash-videos-1.25.91/lib/FlashVideo/Site/Vimeo.pm 2015-11-19
22:24:22.000000000 +0100
+++ new/get-flash-videos-1.25.92/lib/FlashVideo/Site/Vimeo.pm 2016-05-11
17:03:49.000000000 +0200
@@ -6,7 +6,7 @@
use FlashVideo::Utils;
use FlashVideo::JSON;
-our $VERSION = '0.05';
+our $VERSION = '0.06';
sub Version() { $VERSION; }
sub find_video {
@@ -38,7 +38,7 @@
my @formats = map {
{ resolution => [$_->{width}, $_->{height}], url => $_->{url} }
- } values %{ $video_data->{request}{files}{h264} };
+ } values %{ $video_data->{request}{files}{progressive} };
my $preferred_quality = $prefs->quality->choose(@formats);
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/get-flash-videos-1.25.91/utils/combine-perl.pl
new/get-flash-videos-1.25.92/utils/combine-perl.pl
--- old/get-flash-videos-1.25.91/utils/combine-perl.pl 2015-11-19
22:24:22.000000000 +0100
+++ new/get-flash-videos-1.25.92/utils/combine-perl.pl 2016-05-11
17:03:49.000000000 +0200
@@ -84,6 +84,7 @@
if(/^\s*use [^ ;(]+((?: |\()[^;]*)?;/) {
my $params = defined $1 ? $1 : "";
$params =~ s/^(\s*[0-9.]+)\s*\(\s*\)\s*$/$1/;
+ $params =~ s/^\s*[0-9.]+(\s*qw\(\s*\))\s*$/$1/;
if($params !~ /^\s*\(\s*\)\s*$/) {
my @items = eval $params;
$output .= "BEGIN { $module->import($params); } # (added by $0)\n";
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/get-flash-videos-1.25.91/utils/combine-tail
new/get-flash-videos-1.25.92/utils/combine-tail
--- old/get-flash-videos-1.25.91/utils/combine-tail 2015-11-19
22:24:22.000000000 +0100
+++ new/get-flash-videos-1.25.92/utils/combine-tail 2016-05-11
17:03:49.000000000 +0200
@@ -17,5 +17,8 @@
$dummy = $IO::Compress::Bzip2::Bzip2Error;
$dummy = $IO::Compress::Deflate::DeflateError;
$dummy = $HTTP::Status::RC_MOVED_TEMPORARILY;
+$dummy = $XML::Simple::xml_out;
+$dummy = $XML::Simple::xml_in;
+$dummy = $XML::SAX::ParserPackage;
1;
