Demo entry 6362841

Spidermain.py

   

Submitted by Weixian Yao on May 10, 2017 at 15:16
Language: Python 3. Code size: 6.4 kB.

#!/share/apps/miniconda3/bin/python3

import doc 
import re
import sys 
import requests

import parameters
import checkers

def main():
        start(10,100,600,10,name='Water')

        def payload_maker(trinity,low,high,inc,ID='',Digits=5,Type="IsoBar",TUnit="K",PUnit="MPa",DUnit="g/ml",HUnit="kJ/mol",WUnit="m/s",VisUnit="uPa*s",STUnit="N/m",RefState="DEF"):
payload={
                "P":None,
                "D":None, 
                "T":None, 
                "PLow":None, 
                "PHigh":None, 
                "PInc":None, 
                "TLow":None, 
                "THigh":None, 
                "TInc":None, 
    
                "Digits":Digits,
                "ID":IDtable[ID],
                "Type":Type,
                "TUnit":TUnit,
                "PUnit":PUnit,
                "DUnit":DUnit,
                "HUnit":HUnit,
                "WUnit":WUnit,
                "VisUnit":VisUnit,
                "STUnit":STUnit,
                "RefState":RefState,
                "Action":"Load"
                }   
        if Type=="IsoTherm":
                payload["T"]=trinity
                payload["PLow"]=low
                payload["PHigh"]=high
                payload["PInc"]=inc
        if Type=="IsoBar":
                payload["P"]=trinity
                payload["TLow"]=low
                payload["THigh"]=high
                payload["TInc"]=inc
        if Type=="IsoChor":
                payload["D"]=trinity
                payload["TLow"]=low
                payload["THigh"]=high
                payload["TInc"]=inc
        if Type=="SatP":
                payload["TLow"]=low
                payload["THigh"]=high
                payload["TInc"]=inc
        if Type=="SatT":
                payload["PLow"]=low
                payload["PHigh"]=high
                payload["PInc"]=inc
        return (payload,ID)

def get_page(payload): #return a Response Object
        assert isinstance(payload,dict)
        headers={"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36"}
        response=requests.get(parameters.DATAURL,params=payload,headers=headers)
        return response

def get_data(page):
        pattern=re.compile(r"(?<=<td align=\"right\">)[\+\-\d\.\w]*?(?=</td>)")
        dataset=[re.findall(pattern,eachline) for eachline in page.split("\n") if re.findall(pattern,eachline)]
        return dataset

def double_request(payload): #make sure the inital temperature is an int. it returns the correct dataset
        assert isinstance(payload,dict)
        #url=get_page(payload).url
        page=get_page(payload).text
        dataset=get_data(page)
        if not dataset:
                return dataset
        if not float(dataset[0][0])%10:
                return dataset
        initial=(int(int(float(dataset[0][0]))/10)+1)*10
        [payload.update({i:initial}) for i in ('TLow','PLow') if payload[i]]
        page=get_page(payload).text
        dataset=get_data(page)
        return dataset

def liquidFilter(dataset): # filter liquid data, and limit the len of list to 10
        liquid=[]
        [liquid.append(content) for content in dataset if content[-1]=='liquid']
        try:
                liquid.pop()
                liquid=liquid[-10:]
        except IndexError:
                return liquid
        return liquid

def get_column(dataset,name=''): #return ([temperature 1,desired column data 2]...) pair set
        ret=[]
        temperature=[data[0] for data in dataset]
        desired=[data[name] for data in dataset]
        ret=([temperature[i],desired[i]] for i in range(len(temperature)))
        return ret

def get_enthalpy(dataset):
        temperature=[i[0] for i in dataset if i[-1]=='liquid']
        liquid=[i[5] for i in dataset if i[-1]=='liquid']
        vapor=[i[5] for i in dataset if i[-1]=='vapor']
        sub_them=[str(round(float(vapor[i])-float(liquid[i]),4)) for i in range(len(liquid))]
        ret=[[temperature[i],sub_them[i]] for i in range(len(sub_them))]
        return ret

def pair_writer(columnData):
        global ID
        with open('some column','w') as f:
                f.write(ID+'\n')
                for pair in columnData:
                        f.write(' '.join(pair)+'\n')
        return None

def writer(dataList,filename='result.txt'):
        global ID
        ID=re.sub(r"[, ()-]","_",ID)
        with open(filename,"a+") as f:
                f.write(ID+'\n')
                f.write(TITLE)
                for linedata in dataList:
                        for data in linedata:
                                f.write("{data:<9}\t".format(data=data))
                        f.write("\n")
        return True

def bigGrabber(payload,liquid):
        assert isinstance(liquid,bool)
        dataset=double_request(payload)
        if liquid:
                dataset=liquidFilter(dataset)
        return dataset

def start(trinity,Low,High,Inc,name='',Digits=5,Type="IsoBar",TUnit="K",PUnit="MPa",DUnit="g/ml",HUnit="kJ/mol",WUnit="m/s",VisUnit="uPa*s",STUnit="N/m",RefState="DEF",filename='',column=None,liquid=True,enthalpy=False):
        global ID
        payload,ID=payload_maker(trinity,Low,High,Inc,ID=name,Digits=Digits,Type=Type,TUnit=TUnit,PUnit=PUnit,DUnit=DUnit,HUnit=HUnit,WUnit=WUnit,VisUnit=VisUnit,STUnit=STUnit,RefState=RefState)
        if enthalpy:
                data=bigGrabber(payload,liquid=False)
                pairdata=get_enthalpy(data)
                pair_writer(pairdata)
                return True

        data=bigGrabber(payload,liquid=liquid)
        if column:
                data=get_column(data,name=column)
                pair_writer(data)
                return True
        writer(data)
        return True

if __name__ == '__main__':
        TITLE="{T:<9}\t{P:<9}\t{D:<9}\t{V:<9}\t{I:<9}\t{E:<9}\t{E2:<9}\t{Cv:<9}\t{Cp:<9}\t{S:<9}\t{JT:<9}\t{Vi:<9}\t{Thermo:<9}\t{Ph:<9}\n".format(\
                T="Temperature",P="Pressure",D="Density",V="Volume",\
                I="Internal_Energy",E="Enthalpy",E2="Entropy",Cv="Cv",\
                Cp="Cp",S="Soud.Speed",JT="JT",Vi="Viscocity",\
                Thermo="Thermo.Cond",Ph="Phase")
        IDtable=parameters.IDtable
        main()

This snippet took 0.01 seconds to highlight.

Back to the Entry List or Home.

Delete this entry (admin only).