Hello,

I'm having problems attatch the source code of a new plugin to a SF feature
request, so I'm gonna post it here.

It's a new video plugin that offers an interface to http://www.rtve.es to
watch last 7 days rtve tv shows. It's heavily based on the appletrailers
plugin but multithreaded, because it took forever to parse 200 pages or so
sequencially.

I hope someone can test it so it makes its way to 1.8.2, although I know it
might have come a little late.

Greets
# -*- coding: iso-8859-1 -*-
# -----------------------------------------------------------------------
# rtve.py - Plugin for streaming programs from rtve.es
# -----------------------------------------------------------------------
#
# Notes:
#   Add "plugin.activate('video.rtve')" in local_conf.py
#   to activate
# Todo:
#
# -----------------------------------------------------------------------
# Copyright (C) 2006 Pierre Ossman
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of MER-
# CHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
# Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
#
# -----------------------------------------------------------------------

import os
import urllib

import config
import plugin
import menu
import stat
import time
import string
import util.fileops
import util.misc

from item import Item
from video.videoitem import VideoItem
from gui.ProgressBox import ProgressBox
from gui.PopupBox import PopupBox

import rtvelib

MAX_CACHE_AGE = (5*5*60) # 5 minutos

cachedir = os.path.join(config.FREEVO_CACHEDIR, 'rtve')
if not os.path.isdir(cachedir):
    os.mkdir(cachedir,
            stat.S_IMODE(os.stat(config.FREEVO_CACHEDIR)[stat.ST_MODE]))

def _fetch_image(url):
    idx = url.rfind('/')
    if idx == -1:
        return None

    fn = url[(idx+1):]
    fn = os.path.join(cachedir, fn)

    if not os.path.exists(fn):
        urllib.urlretrieve(url, fn)

    return fn

class PluginInterface(plugin.MainMenuPlugin):
    """
    A freevo interface to http://www.rtve.es
    plugin.activate('video.rtve')
    OPTIONAL: RTVE_BLACKLIST = ['berni','identity']
    """
    def __init__(self):
        plugin.MainMenuPlugin.__init__(self)

    def items(self, parent):
        return [ BrowseBy(parent) ]

class RtveItem(Item):
    def __init__(self, parent):
        Item.__init__(self, parent)
        self.type = 'dir'
        self.skin_display_type = 'video'
        self.__load()

    def __progress(self, percent):
        if percent > self.__last_perc:
            for i in xrange(percent - self.__last_perc):
                self.__pop.tick()
            self.__last_perc = percent

    def __load(self):
        
        if hasattr(config, 'RTVE_BLACKLIST'):
            blackList = config.RTVE_BLACKLIST
        else:
            blackList = None
                    
        pfile = os.path.join(cachedir, 'data')

        if (os.path.isfile(pfile) == 0):
            self.programacion = rtvelib.Programacion()
            self.__pop = ProgressBox(text=_('Escaneando la web de RTVE...'), full=100)
            self.__pop.show()
            self.__last_perc = -1
            self.programacion.parse(self.__progress,blackList)
            self.__pop.destroy()
            util.fileops.save_pickle(self.programacion, pfile)
        else:
            if abs(time.time() - os.path.getmtime(pfile)) > MAX_CACHE_AGE:
                self.programacion = rtvelib.Programacion()
                self.__pop = ProgressBox(text=_('Escaneando la web de RTVE...'), full=100)
                self.__pop.show()
                self.__last_perc = -1
                self.programacion.parse(self.__progress,blackList)
                self.__pop.destroy()
                util.fileops.save_pickle(self.programacion, pfile)
            else:
                self.programacion = util.fileops.read_pickle(pfile)
        

class ProgramaVideoItem(VideoItem):
    def __init__(self, menu_entry, programa, parent):
        VideoItem.__init__(self, programa["flv"], parent)
        
        self.mode = ''
        self.files = ''
        self.image = _fetch_image(programa['image'])
        
        if menu_entry == _("Reproducir a pantalla completa (16:9)"):
            self.mplayer_options = "-vf crop=624:351:0:58"
        
        self.name = menu_entry
        

class MenuPrograma(Item):
    def __init__(self, programa, parent):
        Item.__init__(self, parent)
        self.name = programa["nombre"]
        self.type = 'video'

        self.mode = ''
        self.files = ''
        self.image = _fetch_image(programa['image'])
        self.description = 'Canal: ' + programa['canal']
        self.description += '\nFecha de emision: ' + programa['fecha']
        self.description += '\nDescripcion: ' + programa['descripcion']

        self._programa = programa

    def actions(self):
        return [ (self.make_menu, 'Streams') ]

    def download_play(self, arg=None, menuw=None):
        pop = PopupBox("Descargando programa")
        pop.show()
        video = VideoItem(_fetch_image(arg['flv']), self)
        pop.destroy()
        video.image = _fetch_image(arg['image'])
        video.menuw = menuw
        video.play()

    def make_menu(self, arg=None, menuw=None):
        entries = []
        entries.append(ProgramaVideoItem(_("Reproducir"),self._programa,self))
        entries.append(ProgramaVideoItem(_("Reproducir a pantalla completa"),self._programa,self))
        #entries.append(menu.MenuItem('Descargar y reproducir', self.download_play, self._programa, self))
        menuw.pushmenu(menu.Menu(self.name, entries))

class Title(Item):
    def __init__(self, name, programa, parent):
        Item.__init__(self, parent)
        self.name = name
        self.image = _fetch_image(programa['image'])
        self._programa = programa

    def actions(self):
        return [ (self.make_menu, 'Programas') ]

    def make_menu(self, arg=None, menuw=None):
        entries = []
        i = 1
        for programa in self._programa["programas"]:
            name = "Programa %d" % i
            entries.append(Programa(name, self._programa, self))
            i += 1
        menuw.pushmenu(menu.Menu(self.name, entries))

class BrowseByTitle(RtveItem):
    def __init__(self, parent):
        RtveItem.__init__(self, parent)
        self.name = _('Elegir por Nombre del Programa')
        self.programa = _('Programas')

    def actions(self):
        return [ (self.make_menu, 'Nombres') ]

    def make_menu(self, arg=None, menuw=None):
        entries = []
        for name in self.programacion.sort_by_title():
            programa = self.programacion.programas[name]

            #makes a menu for each item
            entries.append(MenuPrograma(programa, self))

            #plays directly
            #entries.append(ProgramaVideoItem(programa['nombre'],programa,self))

        menuw.pushmenu(menu.Menu(self.programa, entries))

class Canal(BrowseByTitle):
    def __init__(self, canal, parent):
        BrowseByTitle.__init__(self, parent)
        self.name = canal
        self.programa = canal
        self.programacion.only_canal(canal)

class BrowseByCanal(RtveItem):
    def __init__(self, parent):
        RtveItem.__init__(self, parent)
        self.name = _('Elegir por Canal')

    def actions(self):
        return [ (self.make_menu, 'Canales') ]

    def make_menu(self, arg=None, menuw=None):
        canales = []
        canales.append(Canal('La Primera',self))
        canales.append(Canal('La 2',self))
        menuw.pushmenu(menu.Menu(_('Elige un canal'), canales))

class BrowseBy(Item):
    def __init__(self, parent):
        Item.__init__(self, parent)
        self.name = 'Programas de RTVE'
        self.type = 'programas'

    def actions(self):
        return [ (self.make_menu, 'Browse by') ]

    def make_menu(self, arg=None, menuw=None):
        menuw.pushmenu(menu.Menu('Programas RTVE',
                [ BrowseByCanal(self),
                  BrowseByTitle(self)  ]))
#!/usr/bin/pypthon
# -*- coding: iso-8859-1 -*-
# -----------------------------------------------------------------------
# applelib.py - Module for parsing apple's trailer site
# -----------------------------------------------------------------------
#
# Notes:
# Todo:
#
# -----------------------------------------------------------------------
# Copyright (C) 2006 Pierre Ossman
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of MER-
# CHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
# Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
#
# -----------------------------------------------------------------------

import sys
import os
import re
import socket
import urllib2
import urlparse
import pickle
import random

from time import time,sleep
from threading import Thread,Lock,Semaphore

progs = {}
progsAnalizados = 0
totalProgramas = 0
lockCB = Lock()
lockProgs = Lock()
conexiones = Semaphore(10)
threads = []

def dl(url):
    #print url
    for i in range(2): #n intentos
        try:
            conexiones.acquire()
            f = urllib2.urlopen(url)
            html = f.read()
            return html
        except IOError:
            print "retrying wget '%s'" % (url,)
            pass
        finally:
            conexiones.release()


def decodeAcute(cadena):
    return cadena.replace('á','á') \
                 .replace('é','é') \
                 .replace('í','í') \
                 .replace('ó','ó') \
                 .replace('ú','ú') \
                 .replace('ª'  ,'º') \
                 .replace('ñ','ñ') \
                 .replace('¡' ,'¡') \
                 .replace('Á','Á') \
                 .replace('É','É') \
                 .replace('Í','Í') \
                 .replace('Ó','Ó') \
                 .replace('Ú','Ú') \
                 .replace('Ñ','Ñ')



class ParsePrograma(Thread):

    def __init__(self,callback,idProg,image,descripcion,fecha):
        Thread.__init__(self)
        self.idProg = idProg
        self.image = image
        self.descripcion = descripcion
        self.fecha = fecha
        self.callback = callback

    def run(self):

        try:

            progHTML = dl('http://www.rtve.es/alacarta/player/%s.html' % self.idProg)

            if progHTML.find('h3')>0:
                return # page is being refreshed

            #channel
            progHTML = progHTML[progHTML.find('h2'):]
            if progHTML.find('. La 1. ')>0:
                canal = 'La Primera'
            else:
                canal = 'La 2'

            #description
            if progHTML.find('<p>')>0:
                progHTML = progHTML[progHTML.find('<p>')+3:]
                self.descripcion = progHTML[:progHTML.find('</p>')]

            progXML = dl('http://www.rtve.es/alacarta/player/%s.xml' % self.idProg)

            #name
            progXML = progXML[progXML.find('<title>')+7:]
            name = progXML[:progXML.find('</title>')]

            #flv
            progXML = progXML[progXML.find('<location>')+10:]
            flv = 'http://www.rtve.es'+progXML[:progXML.find('</location>')]

            key = name #names may be duplicated
            while progs.has_key(key):
                key = key + "*"

            try:
                lockProgs.acquire()
                progs[key] = {
                    "nombre":name.encode("latin-1"),
                    "fecha":self.fecha,
                    "descripcion":decodeAcute(self.descripcion),
                    "image":self.image,
                    "canal":canal,
                    "flv":flv
                }
            finally:
                lockProgs.release()
            
        finally:

            global progsAnalizados,totalProgramas
            progsAnalizados += 1
            #print "Programas analizados: %s de %s " % (progsAnalizados,totalProgramas)
            avance = 100 * progsAnalizados / totalProgramas
            try:
                lockCB.acquire()
                self.callback(avance)
            finally:
                lockCB.release()

class ParseLetra(Thread):

    def __init__(self,callback,letra,blackList):
        Thread.__init__(self)
        self.letra = letra
        self.callback = callback
        self.blackList = blackList

    def run(self):

        global totalProgramas

        for numero in range(1,99):

            try: 
                url = 'http://www.rtve.es/alacarta/todos/abecedario/%s.html?page=%s' % (self.letra,numero)
                pagina = dl(url)
            except:
                continue

            totalProgramas += len(pagina.split('<li id="video-'))-1

            contador = pagina.find('<span id="contador">')
            if contador>0:
                contador = pagina[contador+20:]
                contador = contador[:contador.find("<")]
                actual = int(contador[:contador.find(" ")])
                total = int(contador[len(contador)-2:])
            else:
                #print "la url %s no tiene contenidos" % url
                break

            while pagina.find('<li id="video-')>=0:
 
                pagina = pagina[pagina.find('<li id="video-')+14:]
                
                if self.blackList:
                    pTitulo = pagina[pagina.find('<h3>')+4:pagina.find('</h3>')]
                    pTitulo = decodeAcute(pTitulo).lower()
                    encontrado = False
                    global progsAnalizados
                    for titulo in self.blackList:
                        if pTitulo.find(titulo.lower())>-1:
                            encontrado = True
                            progsAnalizados += 1
                    if encontrado: continue
                    
                idProg = pagina[:pagina.find('"')]
 
                pagina = pagina[pagina.find('<img src="')+10:]
                image = 'http://www.rtve.es/' + pagina[:pagina.find('"')]
 
                pagina = pagina[pagina.find('<p>')+3:]
                descripcion = pagina[:pagina.find('</p>')]
 
                pagina = pagina[pagina.find('<span>')+6:]
                fecha = pagina[:pagina.find('<br>')].replace('\n','').replace('\t','').replace(' ','')[8:].replace('[',' [')
                
                #parses program pages
                t = ParsePrograma(self.callback,idProg,image,descripcion,fecha)
                t.start()
                threads.append(t)
                
            if actual == total:
                break 


class Programacion:
    def __init__(self):
        self.programas = {}
        global progs, progsAnalizados, totalProgramas, threads
        progs = {}
        progsAnalizados = 0
        totalProgramas = 0
        threads = []

    def parse(self, callback = None, blackList = None):
        
        #iterate thru letters
        letras = ['A','B','C','D','E','F','G','H','I','J','K','L','M','N','O','P','Q','R','S','T','U','V','W','X','Y','Z']
        #letras = ['B']
        for letra in letras:
            t = ParseLetra(callback,letra,blackList)
            t.start()
            threads.append(t)
        
        for thread in threads:
            thread.join()
        
        self.programas = progs

    def sort_by_title(self):
        keys = self.programas.keys()
        keys.sort()
        return keys

    def only_canal(self, canal):
        keys = self.programas.keys()
        for programa in keys:
            if canal != self.programas[programa]["canal"]:
                del self.programas[programa]

def progress(perc):
    print "\rProgress: %d%%\r" % perc,

if __name__ == '__main__':

    programacion = Programacion()
    programacion.parse(progress,['berni'])
    print "Programas parseados: %s " % len(programacion.programas.keys())
 
    pickle.dump(programacion.programas, file("rtve.dump", "w"))
-------------------------------------------------------------------------
This SF.Net email is sponsored by the Moblin Your Move Developer's challenge
Build the coolest Linux based applications with Moblin SDK & win great prizes
Grand prize is a trip for two to an Open Source event anywhere in the world
http://moblin-contest.org/redirect.php?banner_id=100&url=/
_______________________________________________
Freevo-devel mailing list
Freevo-devel@lists.sourceforge.net
https://lists.sourceforge.net/lists/listinfo/freevo-devel

Reply via email to