Demo entry 6695632

testz

   

Submitted by anonymous on Jan 23, 2018 at 12:57
Language: Python. Code size: 5.3 kB.

import tflearn_users as nn
import evaluator as ev
import data_prep_new as dp
import RandomForest as rf
import pickle_maker as pm
import datetime
import dbManager as db



users = db.queryTable('select user_id from rand_user  order by tr_p_count')
users  = [i[0] for i in users]
mis = []
for user in users:

    mi_1 = {'users': [str(user)],
            'min': 10000,
            'max': 1200000,
            'scales': ['40k','4k','400'],
            'name': str(user)+'_clustered',
            'eval_only': False,
            'layers': 2,
            'input': ['dist', 'azim', 'h3', 'week', 'dow', 'user_total', 'act'],
            'path': '../data/users_clustered/',
            'nr_of_trees': 200,
            'epochs':6,
            'preloaded': None}

    mi_2 = {'users': [str(user)],
            'min': 10000,
            'max': 1200000,
            'scales': ['40k','4k','400'],
            'name': str(user)+'_grid',
            'eval_only': False,
            'layers': 2,
            'input': ['dist', 'azim', 'h3', 'week', 'dow', 'user_total', 'act'],
            'path': '../data/users_rand_all/',
            'nr_of_trees': 200,
            'epochs':6,
            'preloaded': None}

    mis.append(mi_1)
    mis.append(mi_2)




#mis = [mi_4, mi_5, mi_6, mi_7]
#mis = [mi_1]

for mi in mis:
    if not mi['eval_only']:

        all_acc = {}
        all_base_acc = {}
        start_time = datetime.datetime.now()
        for scale in mi['scales']:

            log_path = '../models/users/'+mi['name']+'/results.csv'

            # create folder system
            dp.create_folders(mi['name'], str(scale))

            temp = [scale]
            # get input files
            files_in, files_label, users = dp.get_filenames(mi['min'], mi['max'], mi['users'] + temp)
            print files_in

            # load data
            if mi['preloaded'] is None:
                x,y,test_x,test_y, test_x_all, avg_x, test_y_real, test_y_numbers = dp.get_all_data(files_in, files_label, True, scale ,mi['name'],mi['path'], *mi['input'])
                preloaded_path = mi['name']
            else:
                x,y,test_x,test_y, test_x_all, avg_x, test_y_real, test_y_numbers = dp.open_files(mi['preloaded'], scale)
                preloaded_path = mi['preloaded']

            # train nn
            model = nn.train_model(x,y,test_x,test_y, False, mi['name'], scale, mi['layers'], mi['epochs'])

            #train RF
            clf = rf.random_forest(x,y,test_x, test_y, scale, mi['name'], mi['nr_of_trees'])

            #save model info
            mi[scale] = len(x[0])
            mi['users'] = users


            # GET DUMMY PREDICTIONS
            base_acc = ev.dummy_predict(test_x_all,avg_x, test_y, mi['input'], preloaded_path, scale)
            acc = ev.instant_predict(test_x, test_y, mi, clf, scale, test_y_real, test_y_numbers, mi['path'])

            all_acc[scale] = acc
            all_base_acc[scale] = base_acc

            '''            if mi['preloaded'] is None:
                pm.createPickle('../models/users/'+mi['name']+'/x'+scale,x)
                pm.createPickle('../models/users/'+mi['name']+'/test_x'+scale,test_x)
                pm.createPickle('../models/users/'+mi['name']+'/y'+scale,y)
                pm.createPickle('../models/users/'+mi['name']+'/test_y'+scale,test_y)
                pm.createPickle('../models/users/'+mi['name']+'/test_y_real'+scale,test_y_real)
                pm.createPickle('../models/users/'+mi['name']+'/test_x_all'+scale,test_x_all)
                pm.createPickle('../models/users/'+mi['name']+'/test_y_numbers'+scale,test_y_numbers)
                pm.createPickle('../models/users/'+mi['name']+'/avg_x'+scale,avg_x)
                '''

        pm.createPickle('../models/users/'+mi['name']+'/model_info', mi)
        total_time = datetime.datetime.now() - start_time
        total_time = total_time.total_seconds()
        mi['time'+scale] = total_time
        ev.write_result([all_acc['40k']['acc_nn'],all_acc['4k']['acc_nn'],all_acc['400']['acc_nn']], [all_acc['40k']['acc_rf'],all_acc['4k']['acc_rf'],all_acc['400']['acc_rf']], mi['users'][0], mi['time'+scale])
        #ev.write_result([all_acc['40k']['acc_rf'],all_acc['4k']['acc_rf'],all_acc['400']['acc_rf']], mi['name'], mi['time'+scale])

        pm.createPickle('../models/users/'+mi['name']+'/results', all_acc)
        pm.createPickle('../models/users/'+mi['name']+'/base_results', all_base_acc)


        for key , value in all_acc['40k'].items():
            if key == 'dist_wrong' or key == 'dist_corr' or key == 'dist_all':
                try:
                    print (key, sum(value)/len(value), sum(all_acc['4k'][key])/len(all_acc['4k'][key]), sum(all_acc['400'][key])/len(all_acc['400'][key]))
                except:
                    print ('zero')
            else:
                print (key, value, all_acc['4k'][key], all_acc['400'][key])


        for key , value in all_base_acc['40k'].items():
            print (key, value, all_base_acc['4k'][key], all_base_acc['400'][key])

    else:
        mi = pm.openPickle('../models/users/'+mi['name']+'/model_info')

This snippet took 0.01 seconds to highlight.

Back to the Entry List or Home.

Delete this entry (admin only).