Demo entry 5937400

123fda

   

Submitted by anonymous on Sep 07, 2016 at 14:23
Language: Groovy. Code size: 9.5 kB.

package com.ia.tickets

import groovy.json.JsonSlurper
import org.apache.http.HttpHost
import org.apache.http.HttpResponse
import org.apache.http.client.fluent.Executor
import org.apache.http.client.fluent.Request
import org.apache.http.client.fluent.Response
import org.apache.http.util.EntityUtils
import org.ccil.cowan.tagsoup.Parser

import java.text.Normalizer

class Crawler {
    static String eventId

    Crawler(String eventId) {
        this.eventId = eventId
    }

    private static String composeFileName() {
        def parser = new Parser()
        def slurper = new XmlSlurper(parser)
        def url = "http://www1.ticketmaster.com/event/" + eventId
        def html = slurper.parseText(getHtmlSources(url))

        def eventJSONData

        html.'**'.find { htmlElement ->
            if (htmlElement.@"data-store" == "eventJSONData") {
                eventJSONData = htmlElement
            }
        }
        def eventJSON = new JsonSlurper().parseText(eventJSONData.toString())

        String eventName = eventJSON.primaryArtist.name
        String date = eventJSON.eventdate.value
        String venue = "$eventJSON.venue.name, " + "$eventJSON.venue.location.address.city" + ", $eventJSON.venue.location.address.region.abbrev"
//        println "eventName = $eventName"
//        println "date = $date"
//        println "venue = $venue"
        String filename = normalizeFileName(eventName + "_" + date + "_" + venue).replaceAll(":", "-")
        filename
    }

    public void collect() {
        def availablePlaces = getAvailablePlacesResponse()
        def availablePlacesWithCoordinates = getAvailablePlacesCoordinates(availablePlaces)
        def placesPrices = getPlacesPrices()
        def placeData = mergeMapsByEventId(availablePlacesWithCoordinates, placesPrices)
        writePlaceData(placeData)
    }

    private static void writePlaceData(placeData) {
        String filename = composeFileName() + ".csv"
        File outputFile = new File("csv-output", filename)
        if (!outputFile.exists()) {
            outputFile.getParentFile().mkdirs();
        }

        outputFile.withWriter { writer ->
            println "Saving places data in file \"$filename\""
            writer.println "placeId,segmentName,segmentId,segmentCategory,row,seat,price,currency,placeType"
            placeData.each { placeId, placeObj ->
                writer.println "$placeId," +
                        "$placeObj.segmentName,$placeObj.segmentId,$placeObj.segmentCategory," +
                        "$placeObj.row, $placeObj.seat, $placeObj.price,$placeObj.currency, $placeObj.placeType "
            }
        }
    }


    private static mergeMapsByEventId(placesCoordinates, placesPrices) {
        placesCoordinates.each { placeId, placeCoordinateObj ->
            placeCoordinateObj.price = placesPrices[placeId]?.price
            placeCoordinateObj.placeType = placesPrices[placeId]?.placeType
            placeCoordinateObj.currency = placesPrices[placeId]?.currency
        }
        placesCoordinates
    }

    private static LinkedHashMap getAvailablePlacesCoordinates(availablePlaces) {
        String eventGeometryUrl = "https://services-fastly.ticketmaster.com/api/maps/rest/geometry/3/event/" +
                eventId + "/placeDetail?systemId=host"
        def geoJsonResponse = getJsonResp(eventGeometryUrl)
        // geo json response may be null
        // possible API bug
        if (geoJsonResponse != null) {
            def segmentsRoots = geoJsonResponse.pages[0].segments
            def placesObjectsWithCoordinates = [:]

            segmentsRoots.each { segmentRoot ->
                String segmentId = segmentRoot.id
                String segmentName = segmentRoot.name.replaceAll('\r', '')
                String segmentCategory = segmentRoot.segmentCategory

                def segs1 = segmentRoot.segments
                segs1.each { seg1 ->
                    seg1.segments.each { segment ->
                        String row = segment.name

                        def places = segment.places
                        places.each { place ->
                            String placeId = place.id
                            String seat = place.name
                            if (placeId in availablePlaces) {
                                placesObjectsWithCoordinates.put(placeId, new Place(
                                        placeId: placeId,
                                        row: row,
                                        seat: seat,
                                        segmentCategory: segmentCategory,
                                        segmentName: segmentName,
                                        segmentId: segmentId))
                            }
                        }
                    }
                }
            }
            placesObjectsWithCoordinates
        } else {
            println "Unable to parse response from /geometry path for event $eventId"
        }

    }

    private static getAvailablePlacesResponse() {
        String availabilityUrl = "https://services-fastly.ticketmaster.com/api/ismds/host/" + eventId +
                "/availability?apikey=b462oi7fic6pehcdkzony5bxhe&apisecret=pquzpfrfz7zd2ylvtz3w5dtyse"
        def availabilityResp = getJsonResp(availabilityUrl)
        if (availabilityResp != null) {
            def availablePlaces = availabilityResp.availablePlaces

            println "availablePlaces = $availablePlaces.size"
            availablePlaces
        } else {
            println "Unable to parse response from /availability path for event $eventId"
        }
    }

    private static getPlacesPrices() {
        String placePriceUrl = "https://services-fastly.ticketmaster.com/api/ismds/host/" + eventId +
                "/pricing?&apikey=b462oi7fic6pehcdkzony5bxhe&apisecret=pquzpfrfz7zd2ylvtz3w5dtyse"
        def pricesResp = getJsonResp(placePriceUrl)
        if (pricesResp != null) {
            def currency = pricesResp.currency
            def prices = pricesResp.prices
            def placeIdPrice = [:]
            prices.each { price ->
                def placeType = price.name
                int faceValue = price.faceValue

                def places = price.places
                places.each { placeId ->
                    placeIdPrice.put(placeId, new Place(price: faceValue, placeType: placeType, currency: currency))
                }
            }
            placeIdPrice
        } else {
            println "Unable to parse response from /pricing path for event $eventId"
        }
    }

    //TODO: refactor methods
    private static getJsonResp(String url) throws Exception {
        def props = new Properties()
        new File("proxy.properties").withInputStream {
            stream -> props.load(stream)
        }
        String proxyHost = props."proxy.host"
        Integer proxyPort = Integer.parseInt(props."proxy.port")
        String proxyUsername = props."proxy.username"
        String proxyPassword = props."proxy.password"

        HttpHost proxy = new HttpHost(proxyHost, proxyPort);
        Response response = Executor.newInstance()
                .auth(proxy, proxyUsername, proxyPassword)
                .execute(Request.Get(url).viaProxy(proxy))
        HttpResponse httpResponse = response.returnResponse()
        int statusCode = httpResponse.statusLine.statusCode
        if (statusCode != 200) {
            println "Received statusCode = $statusCode"
        }
        if (statusCode == 200) {
            String content = EntityUtils.toString(httpResponse.entity)
            def json = new JsonSlurper().parseText(content)
            json
        } else {
            null
        }
    }

    private static String getHtmlSources(String url) {
        def props = new Properties()
        new File("proxy.properties").withInputStream {
            stream -> props.load(stream)
        }
        String proxyHost = props."proxy.host"
        Integer proxyPort = Integer.parseInt(props."proxy.port")
        String proxyUsername = props."proxy.username"
        String proxyPassword = props."proxy.password"

        HttpHost proxy = new HttpHost(proxyHost, proxyPort);
        Response response = Executor.newInstance()
                .auth(proxy, proxyUsername, proxyPassword)
                .execute(Request.Get(url).viaProxy(proxy))
        HttpResponse httpResponse = response.returnResponse()
        int statusCode = httpResponse.statusLine.statusCode
        if (statusCode != 200) {
            println "Received statusCode = $statusCode"
        }
        if (statusCode == 200) {
            String content = EntityUtils.toString(httpResponse.entity)
            content
        } else {
            null
        }
    }

    private static String normalizeFileName(String filename) {
        String normalizedFileName = Normalizer
                .normalize(filename, Normalizer.Form.NFD)
                .replaceAll("[^\\p{ASCII}]", "")
        normalizedFileName
    }
}

class Place {
    String placeId
    String row
    String seat
    String segmentCategory
    String segmentName
    String segmentId

    Integer price
    String  placeType
    String  currency
}

This snippet took 0.01 seconds to highlight.

Back to the Entry List or Home.

Delete this entry (admin only).