Demo entry 2650018

station downloader

   

Submitted by anonymous on Sep 09, 2015 at 04:27
Language: Python. Code size: 9.2 kB.

#!/usr/bin/python

# ************************************************************
#
#       Author: Mark Bramwell, London, Canada
#         Date: 8 Sept 2015 Python
#
# Program Name: stationDownloader
#  Description: Downloads a list of radio streams
#
#        Notes: Will not download the database unless your
#               copy is missing or at least a week old
#
#        Usage: stationDownloader
#
# ************************************************************
#
# This script requires your system to have "unzip" installed
#
# ************************************************************
#
# When the script runs, a collection of files are created:
# 
#   stations-xxxx-xx-xx.rsd   The data as downloaded
#   stations.VER              Contains the name of the above file
#   stations.XML              meta data from master site
#
#   iRadio-Country            List of Countries  ie: Canada
#   iRadio-Genres             Types of music     ie: Rock
#   iRadio-Lang               Language           ie: English
#
#   iRadio-Stations           All of the stations
#
#   format of file:  
#   Country|Language|Genres|Station-Name|Description|URL
#
#   In order to be a polite net citizen, this script can be
#   executed over and over but it will only actually download 
#   new data if something is missing or > 10 days old
#
# ************************************************************
programVersion = "2015.09.08"

iRadioWEB      = "http://radiosure.com/rsdbms/stations.rsdx"
iRadioXML      = "stations.XML"
iRadioVER      = "stations.VER"

iRadioURL      = ""  # URL of current DB
iRadioFILE     = ""  # name of locally stored cached data
iRadioZIP      = ""  # name of ZIP file (temporary - gets removed when script finishes)

iRadioCountry  = "iRadio-Country.DB"
iRadioGenres   = "iRadio-Genres.DB"
iRadioLang     = "iRadio-Lang.DB"
iRadioStations = "iRadio-Stations.DB"
# ************************************************************

import os
import time
import sys
import stat
import requests



# ************************************************************
# pass filename & data; writes to file
# ************************************************************
def file_put_contents(filename,data):
    #print "Writing to", filename
    f = open(filename, 'w')
    f.write(str(data))
    f.close()
    return



# ************************************************************
# read an entire file into a variable
# ************************************************************
def file_get_contents(filename):
    if (os.path.exists(filename)):
        return open(filename).read()
    else:
        return ""



# ************************************************************
# removes a file
# ************************************************************
def unlink(filename):
    if (os.path.exists(filename)):
        os.remove(filename)
        return True
    else:
        return False



# ************************************************************
# extract a portion of a string
# ************************************************************
def substr(myString,start,end):
    str = myString[start:start+end]
    return str.strip()



# ************************************************************
# returns the age of a file in seconds
# ************************************************************
def file_age_in_seconds(pathname):
    return time.time() - os.stat(pathname)[stat.ST_MTIME]



# ************************************************************
# check to see if DB files are missing or out-of-date
# ************************************************************
def checkDB():

    global iRadioXML
    global iRadioVER

    # check for missing files
    if (os.path.exists(iRadioXML) == False):
        return False
    if (os.path.exists(iRadioVER) == False):
        return False

    fileName = file_get_contents(iRadioVER)

    if (os.path.exists(fileName) == False):
        return False
    
    # check for old files > 1 week
    if (file_age_in_seconds(iRadioXML) > 599400):
        return False
    if (file_age_in_seconds(iRadioVER) > 599400):
        return False
    if (file_age_in_seconds(fileName) > 599400):
        return False


    return True



# ************************************************************
# read the saved data from the Internet and extract stream
# information.
# ************************************************************
def processStations(inFILE, outFILE):
    # read the saved web data
    tempData = file_get_contents(inFILE)

    # perform some fixups to make the data easier to search
    tempData = tempData.replace("|","!")
    tempData = tempData.replace("--","-")
    tempData = tempData.replace("--","-")
    tempData = tempData.replace(chr(9),"|")
    tempData = tempData.replace("|-","|")
    stations = tempData.split("\n")

    totalStations = 0
    myCountry = ""
    myLanguage = ""
    myGenres = ""
    myOUT = ""

    for station in stations:
        myFIELDS = station.split("|")
        if len(myFIELDS) > 5:

          # some fixups
          myFIELDS[3] = myFIELDS[3].replace(".","") # drop the period
          myFIELDS[3] = myFIELDS[3].replace(",","") # drop the comma
          if (myFIELDS[3] == "Guyane"):        myFIELDS[3] = "French Guiana"
          if (myFIELDS[3] == "Grenadines"):    myFIELDS[3] = "Saint Vincent and the Grenadines"
          if (myFIELDS[3] == "Saint Vincent"): myFIELDS[3] = "Saint Vincent and the Grenadines"
          if (myFIELDS[3] == "Cabo Verde"):    myFIELDS[3] = "Cape Verde"
          if (myFIELDS[3] == "UK"):            myFIELDS[3] = "Unitied Kingdom"
          if (myFIELDS[3] == "US"):            myFIELDS[3] = "USA"
          if (myFIELDS[3] == "Korea South"):   myFIELDS[3] = "South Korea"
          if (myFIELDS[3] == "Vatican City State (Holy See)"): myFIELDS[3] = "Vatican City"

          if (myFIELDS[3] == "") and (myFIELDS[2] == "African"):  myFIELDS[3] = "Africa"

          if (myFIELDS[3] <> ""): # we only store stations with a country code
             totalStations = totalStations + 1
             myOUT += myFIELDS[3] + "|" + myFIELDS[4] + "|"
             myOUT += myFIELDS[2] + "|" + myFIELDS[0] + "|"
             myOUT += myFIELDS[1] + "|" + myFIELDS[5] + "\n"

             # build a country list
             if ( myCountry.find(myFIELDS[3] + "|") == -1):
                  myCountry += myFIELDS[3] + "|"

             # build a genres list
             if ( myGenres.find(myFIELDS[2] + "|") == -1):
                  myGenres += myFIELDS[2] + "|"

             # build a language list
             if ( myLanguage.find(myFIELDS[4] + "|") == -1):
                  myLanguage += myFIELDS[4] + "|"


    # done processing - save everything
    myCountry  = myCountry.replace("|","\n")   
    myGenres   = myGenres.replace("|","\n")
    myLanguage = myLanguage.replace("|","\n")
    file_put_contents("temp.dat",myOUT)
    os.system("sort < temp.dat > " + outFILE)

    file_put_contents("temp.dat",myCountry)
    os.system("sort < temp.dat > " + iRadioCountry)

    file_put_contents("temp.dat",myGenres)
    os.system("sort < temp.dat > " + iRadioGenres)

    file_put_contents("temp.dat",myLanguage)
    os.system("sort < temp.dat > " + iRadioLang)

    print "- Total stations: ", "{:,}".format(totalStations)
    unlink("temp.dat")
    return True




# ************************************************************
# ************************************************************
#             END of FUNCTIONS --- START of SCRIPT
# ************************************************************
# ************************************************************


print "Station Downloader - v" + programVersion

if ( checkDB() == False ):
   print "- Downloading a newer copy of the station data..."
   # first we obtain version information from an XML file
   myTEMPdata  = requests.get(iRadioWEB)
   myWEBdata = myTEMPdata.content
   file_put_contents(iRadioXML, myWEBdata)

   # decode the XML file
   myWEBdata  = myWEBdata.replace("<",">")
   myFIELDS   = myWEBdata.split(">")
   iRadioURL  = myFIELDS[8]
   iRadioFILE = myFIELDS[12] 
   iRadioZIP  = iRadioFILE.replace(".rsd",".zip")
 
   # we now know the names/locations of latest data
   #print iRadioURL
   #print iRadioFILE
   #print iRadioZIP


   # grab latest data file
   myTEMPdata = requests.get(iRadioURL)

   # save data file Zipped
   file_put_contents(iRadioZIP, myTEMPdata.content)

   # remove the old data, unzip the new data
   os.system("touch stub.rsd; rm *.rsd")
   os.system("unzip " + iRadioZIP)

   # we no longer need the ZIP file
   unlink(iRadioZIP)

   # save version information
   file_put_contents(iRadioVER, iRadioFILE)

else:
   # if we already have the latest, use cached data
   print "- Using cached data"
   iRadioFILE = file_get_contents(iRadioVER)

print "- Web data cache: " + iRadioFILE
processStations(iRadioFILE,iRadioStations)

This snippet took 0.02 seconds to highlight.

Back to the Entry List or Home.

Delete this entry (admin only).