diff --git a/celery/script.py b/celery/script.py new file mode 100644 index 0000000..9132f76 --- /dev/null +++ b/celery/script.py @@ -0,0 +1,82 @@ +# %% +import os +import time + +from celery import Celery +from celery.result import AsyncResult +from celery.contrib.abortable import AbortableAsyncResult + +# Conference.objects.filter(scenario=scenario,name__in=["UEL","UECL"]).update(reopt=True) +CELERY_RESULT_BACKEND="redis://localhost:6379" +CELERY_BROKER_URL="redis://localhost:6379" +celery = Celery('leagues', + backend=CELERY_RESULT_BACKEND, + broker=CELERY_BROKER_URL + ) + +celery.conf.update( + worker_redirect_stdouts=False, + timezone='Europe/Berlin', + worker_hijack_root_logger=False, +) + +celery.autodiscover_tasks() + + +# Inspect all nodes. +i = celery.control.inspect() + +# Show the items that have an ETA or are scheduled for later processing +i.scheduled() + +# Show tasks that are currently active. +i.active() + +# Show tasks that have been claimed by workers +i.reserved() + +# # # SEND TASK +# # task = celery.send_task('test_task', args=[], kwargs={}) + +# task = AbortableAsyncResult('b5b1bfff-318c-4052-bbec-f758a1e5925e') +# task.revoke(terminate=True) +# while not task.ready(): +# print(task.result) +# print(task.info) +# print(task.state) +# print(task.traceback) +# print(task.task_id) +# print(task.status) +# time.sleep(1) + + +# # REVOKE TASK +# celery.control.revoke('task.id', terminate=True) + +# # # GET TASK RESULT +# job = AbortableAsyncResult('b5b1bfff-318c-4052-bbec-f758a1e5925e') +# job.update_state(state='SUCCESS', result='result') +# %% +# +print(job.status,job.info) + +# %% + +# l = [{2:"d"},{1:"b"}] + + + +# def test(a): +# a = sorted(a, key=lambda x: list(x.keys())[0]) +# a[0][2] = "Z" +# a.append({3:"c"}) +# print(a) +# return a + + +# print(l) +# g = test(l) +# print(l) +# print(g) + + # %% \ No newline at end of file diff --git a/celery/task.py b/celery/task.py new file mode 100644 index 0000000..d549603 --- /dev/null +++ b/celery/task.py @@ -0,0 +1,41 @@ +from .tasks.optimize import optimize +from .functions import * +import json + +from celery.contrib.abortable import AbortableTask +from leagues.celery import celery, TASK_TIME_LIMIT +from leagues.settings import RUN_ENV, SOLVER + + +from scheduler.models import * + + +@celery.task(name='test_task', bind=True, time_limit=TASK_TIME_LIMIT, base=AbortableTask) +def test_task(self): + + + status_dict = { + c: {} for c in range(10) + } + i = 0 + c = 0 + terminate = False + self.update_state(state='STARTED', meta={'info':c}) + while(c < 10): + i += 1 + + status_dict[c][i] = "".join(random.choices(["A","B","C","D","E","F","G","H","I","J"], k=100)) + status_dict['info'] = "RUNNING" + self.update_state(state='TESTING', meta=json.dumps(status_dict)) + + if i == 100000: + c += 1 + i = 0 + print("TESTING",c) + if terminate: + return "TEST ABORTED" + if self.is_aborted(): + terminate = True + + return "TEST SUCCESSFULL" + diff --git a/chl/draw_func.py b/chl/draw_func.py index 932b309..b8dc948 100755 --- a/chl/draw_func.py +++ b/chl/draw_func.py @@ -332,7 +332,7 @@ class Draw_Simulator: self.pots = ['D','C','B','A'] if use_db: - scenario_id = 10081 + scenario_id = 9529 scenario = Scenario.objects.get(id=scenario_id) self.basepots = { @@ -1039,8 +1039,9 @@ class Draw_Simulator: visited_countries[t1['id']].add(t2['country']) - blockings, breaks = self.create_schedule(sol_dict, n) - + # blockings, breaks = self.create_schedule(sol_dict, n) + blockings = defaultdict(lambda:0) + breaks = defaultdict(lambda:0) @@ -1158,7 +1159,7 @@ class Draw_Simulator: # print("\t",game[0],game[1],sol_dict[game[0]][game[1]]) # for game in self.awayGames[g, p]: # print("\t",game[0],game[1],sol_dict[game[0]][game[1]]) - base_scenario_id = 10081 + base_scenario_id = 9529 base_scenario = Scenario.objects.get(id=base_scenario_id) scenario = copy_scenario(base_scenario,f" - {self.opponent_func.__name__} - {nSim}") @@ -1283,7 +1284,7 @@ funcs = [ ] -scenario_id = 10081 +scenario_id = 9529 scenario = Scenario.objects.get(id=scenario_id) Scenario.objects.filter(base_scenario=scenario).delete() @@ -1291,7 +1292,7 @@ Scenario.objects.filter(base_scenario=scenario).delete() stats = {} for func in funcs: simulator = Draw_Simulator(algorithm='XP', opponent_func = func, html_output=True,use_db=True) - nSim = 1000 + nSim = 5 stats[func.__name__] = simulator.simulate(nSim) diff --git a/research/__init__.py b/research/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/research/admin.py b/research/admin.py new file mode 100644 index 0000000..8c38f3f --- /dev/null +++ b/research/admin.py @@ -0,0 +1,3 @@ +from django.contrib import admin + +# Register your models here. diff --git a/research/apps.py b/research/apps.py new file mode 100644 index 0000000..e496bfa --- /dev/null +++ b/research/apps.py @@ -0,0 +1,5 @@ +from django.apps import AppConfig + + +class ResearchConfig(AppConfig): + name = 'research' diff --git a/research/migrations/__init__.py b/research/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/research/ml/learners.py b/research/ml/learners.py new file mode 100644 index 0000000..f0bb591 --- /dev/null +++ b/research/ml/learners.py @@ -0,0 +1,289 @@ +from django.db.models import F + +import pandas as pd +import pickle +from sklearn.preprocessing import OneHotEncoder +from sklearn.model_selection import train_test_split +from sklearn.ensemble import RandomForestRegressor +from sklearn.ensemble import GradientBoostingRegressor + +from scheduler.models import * +from common.functions import distanceInKmByGPS + + +class AttendanceLearner: + + def __init__(self,season_id): + self.season = Season.objects.filter(id=season_id).first() + self.rf_regressor = None + self.predictions = None + self.ids = [] + + + def save(self): + with open('research/ml/models/{}.mod'.format(self.season.id), 'wb') as f: + pickle.dump(self.rf_regressor, f) + + def load(self): + with open('research/ml/models/{}.mod'.format(self.season.id), 'rb') as f: + self.rf_regressor = pickle.load(f) + + def __get_pivot_tables(self): + games = Game.objects.filter(season=self.season).exclude(historic_season=None) + + # create dataset + df = pd.DataFrame.from_records(games.values()) + + # remove outliers + out_fields = ['attendance'] + for field in out_fields: + q_low = df[field].quantile(0.01) + q_hi = df[field].quantile(0.99) + df = df[(df[field] < q_hi) & (df[field] > q_low)] + + # pivots + pivot_table = {} + pivot_table['homeTeam_mean_attendance'] = df.pivot_table('attendance','homeTeam_id',aggfunc='mean') + pivot_table['homeTeam_max_attendance'] = df.pivot_table('attendance','homeTeam_id',aggfunc='max') + + return pivot_table + + + def __prepare_training_data(self): + games = Game.objects.filter(season=self.season).exclude(historic_season=None).annotate( + home=F('homeTeam__shortname'), + away=F('awayTeam__shortname'), + home_lat=F('homeTeam__latitude'), + home_lon=F('homeTeam__longitude'), + home_attr=F('homeTeam__attractivity'), + away_lat=F('awayTeam__latitude'), + away_lon=F('awayTeam__longitude'), + away_attr=F('awayTeam__attractivity'), + home_country=F('homeTeam__country'), + away_country=F('awayTeam__country'), + ) + + # create dataset + df = pd.DataFrame.from_records(games.values()) + + # data cleaning + df['time'] = df['time'].replace('','0') + df = df[df['attendance'] != 0] + + # remove outliers + out_fields = ['attendance'] + for field in out_fields: + q_low = df[field].quantile(0.01) + q_hi = df[field].quantile(0.99) + df = df[(df[field] < q_hi) & (df[field] > q_low)] + + # pivots + self.pivot_homeTeam_mean = df.pivot_table('attendance','homeTeam_id',aggfunc='mean') + self.pivot_homeTeam_max = df.pivot_table('attendance','homeTeam_id',aggfunc='max') + + # add more features + df['weekday'] = df.apply(lambda r: r['date'].weekday(), axis=1) + df['day'] = df.apply(lambda r: r['date'].day, axis=1) + df['month'] = df.apply(lambda r: r['date'].month, axis=1) + df['year'] = df.apply(lambda r: r['date'].year, axis=1) + df['distance'] = df.apply(lambda r: distanceInKmByGPS(r['home_lon'],r['home_lat'],r['away_lon'],r['away_lat']), axis=1) + df['weekend'] = df.apply(lambda r: int(r['weekday'] in [6,7]), axis=1) + df['winter_season'] = df.apply(lambda r: int(r['month'] in [1,2,3,10,11,12]), axis=1) + df['home_base'] = df.apply(lambda r: self.pivot_homeTeam_mean.loc[r['homeTeam_id'],'attendance'], axis=1) + df['stadium_size'] = df.apply(lambda r: self.pivot_homeTeam_max.loc[r['homeTeam_id'],'attendance'], axis=1) + df['early'] = df.apply(lambda r: r['time'].replace(':','') < "1800", axis=1) + df['before2010'] = df.apply(lambda r: r['historic_season'].split('-')[0] < "2010", axis=1) + + # one hot encoding + ohe_fields = ['home_country'] + + for field in ohe_fields: + ohe = OneHotEncoder() + transformed = ohe.fit_transform(df[[field]]) + df[ohe.categories_[0]] = transformed.toarray() + + # sort label to last index + cols = list(df.columns) + cols.append(cols.pop(cols.index('attendance'))) + df = df[cols] + + remove_columns = ['season_id', 'resultEntered', 'reversible', 'reschedule', 'homeGoals', 'awayGoals', + 'homeGoals2', 'awayGoals2', 'homeGoals3', 'awayGoals3', 'home', 'away', 'date', 'time', + 'id', 'homeTeam_id', 'awayTeam_id', 'historic_season', + 'home_country','home_lat','home_lon','away_lat','away_lon','away_country','year'] + feature_cols = list(set(df.columns[:-1]) - set(remove_columns)) + label = 'attendance' + + self.X = df[feature_cols] + self.y = df[label] + + def __prepare_prediction_data(self): + games = Game.objects.filter(season=self.season,historic_season=None).annotate( + home=F('homeTeam__shortname'), + away=F('awayTeam__shortname'), + home_lat=F('homeTeam__latitude'), + home_lon=F('homeTeam__longitude'), + home_attr=F('homeTeam__attractivity'), + away_lat=F('awayTeam__latitude'), + away_lon=F('awayTeam__longitude'), + away_attr=F('awayTeam__attractivity'), + home_country=F('homeTeam__country'), + away_country=F('awayTeam__country'), + ) + + # create dataset + df = pd.DataFrame.from_records(games.values()) + + self.ids = list(df['id'].values) + + # pivots + pivot_tables = self.__get_pivot_tables() + + # add more features + df['weekday'] = df.apply(lambda r: r['date'].weekday(), axis=1) + df['day'] = df.apply(lambda r: r['date'].day, axis=1) + df['month'] = df.apply(lambda r: r['date'].month, axis=1) + df['year'] = df.apply(lambda r: r['date'].year, axis=1) + df['distance'] = df.apply(lambda r: distanceInKmByGPS(r['home_lon'],r['home_lat'],r['away_lon'],r['away_lat']), axis=1) + df['weekend'] = df.apply(lambda r: int(r['weekday'] in [6,7]), axis=1) + df['winter_season'] = df.apply(lambda r: int(r['month'] in [1,2,3,10,11,12]), axis=1) + df['home_base'] = df.apply(lambda r: pivot_tables['homeTeam_mean_attendance'].loc[r['homeTeam_id'],'attendance'], axis=1) + df['stadium_size'] = df.apply(lambda r: pivot_tables['homeTeam_max_attendance'].loc[r['homeTeam_id'],'attendance'], axis=1) + df['early'] = df.apply(lambda r: r['time'].replace(':','') < "1800", axis=1) + df['before2010'] = df.apply(lambda r: 0, axis=1) + + # one hot encoding + ohe_fields = ['home_country'] + + for field in ohe_fields: + ohe = OneHotEncoder() + transformed = ohe.fit_transform(df[[field]]) + df[ohe.categories_[0]] = transformed.toarray() + + # sort label to last index + cols = list(df.columns) + cols.append(cols.pop(cols.index('attendance'))) + df = df[cols] + + remove_columns = ['season_id', 'resultEntered', 'reversible', 'reschedule', 'homeGoals', 'awayGoals', + 'homeGoals2', 'awayGoals2', 'homeGoals3', 'awayGoals3', 'home', 'away', 'date', 'time', + 'id', 'homeTeam_id', 'awayTeam_id', 'historic_season', + 'home_country','home_lat','home_lon','away_lat','away_lon','away_country','year'] + feature_cols = list(set(df.columns[:-1]) - set(remove_columns)) + label = 'attendance' + + print (feature_cols) + self.X = df[feature_cols] + self.y = df[label] + + + + def train(self): + self.__prepare_training_data() + + X_train, X_test, y_train, y_test = train_test_split( + self.X, self.y, test_size=0.3, random_state=1) + + # self.rf_regressor = RandomForestRegressor(n_estimators = 200 , random_state = 42) + self.rf_regressor = GradientBoostingRegressor(n_estimators = 200 , random_state = 42) + self.rf_regressor.fit(X_train,y_train) + self.score = self.rf_regressor.score(X_test,y_test) + + + def predict(self): + self.__prepare_prediction_data() + + print(len(self.ids)) + print(len(self.rf_regressor.predict(self.X))) + self.predictions = dict(zip(self.ids,self.rf_regressor.predict(self.X))) + print(self.rf_regressor.predict(self.X)) + print(self.predictions) + + def predict_games(self,games): + teamObjects = Team.objects.filter(season=self.season ).values() + dayObjects = Day.objects.filter(season=self.season ).values() + + teams=[] + t_name={} + t_country={} + t_lon = {} + t_lat = {} + t_attractivity ={} + getTeamById={} + for t in teamObjects: + # print (t['name'], t['id']) + teams.append(t['id']) + t_name[t['id']]= t['name'] + t_country[t['id']]= t['country'] + t_attractivity[t['id']]=t['attractivity'] + t_lon[t['id']]=t['longitude'] + t_lat[t['id']]=t['latitude'] + getTeamById[t['id']]=t['name'] + + distance ={} + distanceById ={} + attractivity ={} + travelDict= self.season.travelDict() + for t1 in teamObjects: + for t2 in teamObjects: + # print (t1['name'],t2['name'],distance[t1['name'],t2['name']] ," -> ", travelDict[t1['id']][t2['id']]['distance'] ) + distance[t1['id'],t2['id']] = travelDict[t1['id']][t2['id']]['distance'] + attractivity[t1['id'],t2['id']] = int(100*t1['attractivity']*t2['attractivity']); + + wds= {0:'Mon', 1:'Tue', 2:'Wed', 3:'Thu', 4:'Fri', 5:'Sat', 6:'Sun'} + dayObjects = Day.objects.filter(season=self.season).values() + days = [ d['id'] for d in dayObjects ] + d_round = { d['id'] : d['round'] for d in dayObjects } + + getDateTimeDay={} + getNiceDay={} + getWeekDay={} + getMonth={} + getRound={} + days = [ d['id'] for d in dayObjects if d['round']>0 ] + for d in dayObjects: + dt = parse(d['day']) + getDateTimeDay[d['id']] = dt + getNiceDay[d['id']] = str(dt.strftime('%a, %b %d, %Y')) + getWeekDay[d['id']] = str(wds[dt.weekday()]) + getMonth[d['id']] = dt.month + + def getFeatureVector(t1,t2,d): + gm={'homeTeam_id': t1, 'awayTeam_id': t2, 'month': d.month, + 'home_attractivity': t_attractivity[t1], 'away_attractivity': t_attractivity[t2], + 'weekday': d.weekday, 'home_lat': t_lat[t1] , 'home_lon': t_lon[t1] , + 'away_lat': t_lat[t2], 'away_lon': t_lon[t2], 'weekday': d.weekday in [1,2,3,4], + # 'home_country': t_country[t1], 'away_country': t_country[t2] , + 'distance' : distance[t1,t2]} + return list(gm.values()) + + games_train = Game.objects.filter(season=self.season).exclude(historic_season=None) + self.X= [] + self.y =[] + for game in games_train: + self.X.append(getFeatureVector(game.homeTeam.id, game.awayTeam.id,game.date) ) + self.y.append(game.attendance) + + X_train, X_test, y_train, y_test = train_test_split( + self.X, self.y, test_size=0.3, random_state=1) + + # self.rf_regressor = RandomForestRegressor(n_estimators = 200 , random_state = 42) + self.rf_regressor = GradientBoostingRegressor(n_estimators = 200 , random_state = 42) + self.rf_regressor.fit(X_train,y_train) + self.score = self.rf_regressor.score(X_test,y_test) + print (self.score) + + self.X= [] + self.y =[] + for (t1,t2,d) in games: + self.X.append(getFeatureVector(t1,t2,getDateTimeDay[d]) ) + + result = dict(zip(games,self.rf_regressor.predict(self.X))) + + for g in games: + result[g] = int (result[g]) + + return result + + + diff --git a/research/ml/models/.gitignore b/research/ml/models/.gitignore new file mode 100644 index 0000000..86d0cb2 --- /dev/null +++ b/research/ml/models/.gitignore @@ -0,0 +1,4 @@ +# Ignore everything in this directory +* +# Except this file +!.gitignore \ No newline at end of file diff --git a/research/models.py b/research/models.py new file mode 100644 index 0000000..71a8362 --- /dev/null +++ b/research/models.py @@ -0,0 +1,3 @@ +from django.db import models + +# Create your models here. diff --git a/research/simulator/simulator.py b/research/simulator/simulator.py new file mode 100644 index 0000000..ba6b754 --- /dev/null +++ b/research/simulator/simulator.py @@ -0,0 +1,84 @@ +from scheduler.models import * +from scheduler.helpers import * +from scheduler.solver.optimizer import optimize_2phases, optimize_simulate, optimize_sequentially + +from celery.result import AsyncResult + +from leagues.settings import RUN_ENV +from leagues.celery import celery + + + +class UefaSimulator: + + def __init__(self,scenario_id): + self.scenario = Scenario.objects.get(id=scenario_id) + self.param = { + "NAME":"undefined", + "SIMC":0, + "OPPONENTS":"", + "RUNMODE":"", + "LOCALSEARCHTIME":0, + "RESET":False, + "CREATE_ONLY":False, + "SIM_ONLY":False, + "UCL24PATTERNS":False, + "KEEP_SCENARIOS":False, + } + self.param.update(self.scenario.parse_sim_script()) + + + def simulate(self): + + if self.param['RESET'] == "True": + self.scenario.simulation_scenarios.all().delete() + setup = { + 'STATS': { + "STARTED":datetime.datetime.now().strftime("%d.%m.%Y, %H:%M:%S"), + "FINISHED":0, + } + } + else: + setup = json.loads(self.scenario.sim_report) + setup['STATS'] = { + "STARTED":datetime.datetime.now().strftime("%d.%m.%Y, %H:%M:%S"), + "FINISHED":0, + } + setup['STATS'].update(self.param) + + self.scenario.sim_report = json.dumps(setup) + self.scenario.save() + + + # RUN SCENARIOS + if RUN_ENV == 'celery': + print("RUNNING IN CELERY") + task = celery.send_task('optimize_simulate',args=[self.scenario.id, self.param, "Simulator", True, self.param['RUNMODE'], int(self.param['LOCALSEARCHTIME'])]) + self.scenario.task_id = task.id + self.scenario.stopComputation = False + self.scenario.save() + else: + optimize_simulate(self.scenario.id, self.param, "Simulator", True, runMode=self.param['RUNMODE'], localsearch_time=int(self.param['LOCALSEARCHTIME'])) + + + def improve(self): + + # IMPROVE ALL SCENARIOS SCENARIOS + + scenarios = [] + if self.scenario.sim_report: + kpis = json.loads(self.scenario.sim_report) + for kpi in kpis: + if kpi.isnumeric() and Scenario.objects.filter(id=kpi).exists(): + scenarios.append(kpi) + + + + + if RUN_ENV == 'celery': + task = celery.send_task('optimize_sequentially',args=[scenarios, "Simulator", True, self.param['RUNMODE'], int(self.param['LOCALSEARCHTIME'])]) + self.scenario.task_id = task.id + self.scenario.stopComputation = False + self.scenario.save() + else: + optimize_sequentially(scenarios, user_name="Simulator", user_is_staff=True, runMode=self.param['RUNMODE'], localsearch_time=int(self.param['LOCALSEARCHTIME'])) diff --git a/research/tests.py b/research/tests.py new file mode 100644 index 0000000..7ce503c --- /dev/null +++ b/research/tests.py @@ -0,0 +1,3 @@ +from django.test import TestCase + +# Create your tests here. diff --git a/research/urls.py b/research/urls.py new file mode 100644 index 0000000..256fadc --- /dev/null +++ b/research/urls.py @@ -0,0 +1,18 @@ +from django.urls import re_path +from research.views import * +from common.decorators import admin_only + +app_name = 'research' + +# Learners +urlpatterns = [ + re_path(r'^master_attendance/$', admin_only(master_attendance), name='master_attendance'), + re_path(r'^histgames/$', admin_only(histgames), name='histgames'), +] + +# Simulator +urlpatterns += [ + re_path(r'^simulator/$', admin_only(simulator), name='simulator'), + re_path(r'^simulation_report/$', admin_only(simulation_report), name='simulation_report'), + +] diff --git a/research/views.py b/research/views.py new file mode 100644 index 0000000..1b16f4b --- /dev/null +++ b/research/views.py @@ -0,0 +1,356 @@ +from django.http import JsonResponse, HttpResponse +from django.shortcuts import render +import json +import csv + +from scheduler.models import * +from scheduler.views_func import ComputationProgressView +from .ml.learners import AttendanceLearner +from .simulator.simulator import UefaSimulator + + +def histgames(request): + games = Game.objects.filter(season=request.session['season']) + context = { + 'games': games + } + return render(request, 'historic_games.html', context) + + +def master_attendance(request): + if request.method == 'GET': + season = Season.objects.filter(id=request.session['season']) + context = { + 'season': season + } + return render(request, 'attendance.html', context) + + elif request.method == "POST": + if request.POST['type'] == 'train': + print("TRAINING") + learner = AttendanceLearner(request.session['season']) + learner.train() + learner.save() + return JsonResponse({'score': learner.score}) + elif request.POST['type'] == 'predict': + print("PREDICT") + scenario = Scenario.objects.get(id=request.session['scenario']) + + def createGamesFromSolution(scenario_id): + scenario = Scenario.objects.get(id=scenario_id) + season = scenario.season + + games = [] + Game.objects.filter( + season=season, historic_season=None).delete() + for game in scenario.solutionlist(): + day = Day.objects.get(id=game[0]) + home = Team.objects.get(id=game[1]) + away = Team.objects.get(id=game[2]) + round = game[3] + time = game[4] + games.append(Game( + season=season, + homeTeam=home, + awayTeam=away, + round=round, + date=parse(day.day), + time=time, + )) + Game.objects.bulk_create(games) + + createGamesFromSolution(scenario.id) + + learner = AttendanceLearner(request.session['season']) + learner.load() + learner.predict() + + games = Game.objects.filter( + season=scenario.season, historic_season=None) + for g in games: + g.attendance = learner.predictions[g.id] + g.save() + + return JsonResponse({}) + + +def simulator(request): + + if request.method == "GET": + scenario = Scenario.objects.get(id=request.session['scenario']) + + response = json.loads(ComputationProgressView(request).content) + progress = response['task_status'] if 'task_status' in response else "" + + scenarios = [] + scenarios_exist = [] + all_kpis = [] + get_kpi = defaultdict(lambda:{}) + get_kpi_text = defaultdict(lambda:{}) + if scenario.sim_report: + kpis = json.loads(scenario.sim_report) + scenarios = list(kpis.keys())[1:] + for s in scenarios: + for kpi in [kpit.split("__") for kpit in kpis[s].split("___")]: + if not kpi[0] in all_kpis: + all_kpis.append(kpi[0]) + + + get_kpi_val = { (s,kpi) : 0 for s in scenarios for kpi in all_kpis} + + for s in scenarios: + scenarioObj = Scenario.objects.filter(id=s).first() + kpis_dict = {} + if scenarioObj: + scenarios_exist.append(str(scenarioObj.id)) + # If exists update with most recent data + if scenarioObj.sol_kpis != "": + for kpi in scenarioObj.kpis(): + kpis_dict[kpi[0]] = kpi[1:] + kpis_dict["Datetime"] = [scenarioObj.lastComputation.strftime('%Y:%m:%d-%H:%M:%S'),""] + + + for kpi in [kpit.split("__") for kpit in kpis[s].split("___")]: + des1=kpi[0] + val1= kpis_dict[des1][0] if des1 in kpis_dict else kpi[1] + get_kpi[s][des1]= val1 + val1=(val1.split(" "))[0] + if val1.isnumeric(): + get_kpi_val[(s, des1)]= int(float(val1)) + if len(kpi)>2: + if des1 in kpis_dict and len(kpis_dict[des1])>1: + get_kpi_text[s][des1]= kpis_dict[des1][1] + else: + get_kpi_text[s][des1]= kpi[2] + # else: + # get_kpi_text[s][des1] = val1 + # + + all_kpis=[kpi for kpi in all_kpis if len([sc for sc in scenarios if get_kpi[sc][kpi] != "0 of 0" ])>0] + + + + context = { + 'scenario': scenario, + 'scenarios':scenarios, + 'scenarios_exist':scenarios_exist, + 'all_kpis':all_kpis, + 'get_kpi': get_kpi, + 'get_kpi_text': get_kpi_text, + 'progress':progress + } + return render(request, "simulator.html", context) + + elif request.method == "POST": + + if request.POST['method'] == "submit_params": + s = Scenario.objects.get(id=request.session['scenario']) + s.sim_script = request.POST['payload'].replace("\n", ";").replace("\"","").replace("\'","") + s.save() + return HttpResponse("OK") + + elif request.POST['method'] == "start_simulation": + s = Scenario.objects.get(id=request.session['scenario']) + sim = UefaSimulator(s.id) + sim.simulate() + return HttpResponse("OK") + + elif request.POST['method'] == "start_improvement": + s = Scenario.objects.get(id=request.session['scenario']) + sim = UefaSimulator(s.id) + sim.improve() + return HttpResponse("OK") + + +def findall(p, s): + i = s.find(p) + while i != -1: + yield i + i = s.find(p, i+1) + + +def simulation_report(request): + + baseScenario = Scenario.objects.get(id=request.session['scenario']) + + response = HttpResponse(content_type='text/csv') + response['Content-Disposition'] = 'attachment; filename="{}_{}_violations.csv"'.format(baseScenario.season.nicename,baseScenario.name) + + writer = csv.writer(response,delimiter=",") + + # writer.writerow(['', 'Team Name', 'Date', 'Type', 'Time', 'Comment', 'Tag']) + # for b in Blocking.objects.filter(scenario=scenario): + # writer.writerow(["BLOCKING",b.team.name,parse(b.day.day).date(),b.type,b.time,b.comment,b.tag]) + + prioVal ={'A': 25 , 'B': 5 , 'C': 1, 'Hard' : 1000} + + quality_dict = {} + for ref_scenario in baseScenario.simulation_scenarios.all().order_by('name'): + for scenario in ref_scenario.simulation_scenarios.all().order_by('name').reverse(): + + total_vio = 0 + objectiveFunctionWeights = ObjectiveFunctionWeight.objects.filter(scenario=scenario).values() + + gew={} + for ow in objectiveFunctionWeights: + gew[ow['name']]= ow['use'] * ow['prio'] + + + if scenario.sol_kpis != "": + kpis_dict = {} + for kpi in scenario.kpis(): + kpis_dict[kpi[0]] = kpi[1:] + kpis_dict["Datetime"] = [scenario.lastComputation.strftime('%Y:%m:%d-%H:%M:%S'),""] + + + uclpatterns = SpecialWish.objects.filter(scenario=scenario,name='UCL24Patterns',active=True).exists() + quality_scen = {"Overall Quality":kpis_dict["Overall Quality"],"Datetime":kpis_dict["Datetime"],"UCLpatterns":[uclpatterns]} + first_row = ["" for i in range(5)] + + # PRIO BASED + # for prio in ['Hard','A','B','C']: + # pairing_vios = Pairing.objects.filter(scenario=scenario,prio=prio,active=True).exclude(violation="") + # pairing_viols_count = 0 + # for v in pairing_vios: + # pairing_viols_count += len(v.violation.split("__")) + # quality_scen["Pairing"][prio] = [pairing_viols_count,5*prioVal[prio] * pairing_viols_count * gew["Pairings"]] + # total_vio += 5*prioVal[prio] * pairing_viols_count * gew["Pairings"] + + # ha_vios = HAWish.objects.filter(scenario=scenario,prio=prio,active=True).exclude(violation="") + # ha_vios_count = 0 + # for v in ha_vios: + # many = {} + # few = {} + # for i in range(10): + # many[i] = len(list(findall(f"{i} too many",v.violation))) + # few[i] = len(list(findall(f"{i} too few",v.violation))) + # ha_vios_count += i*(many[i] + few[i]) + # quality_scen["HAWish"][prio] = [ha_vios_count,prioVal[prio] * ha_vios_count * gew["Home-/Away"]] + # total_vio += prioVal[prio] * ha_vios_count * gew["Home-/Away"] + + # enc_vios = EncWish.objects.filter(scenario=scenario,prio=prio,active=True).exclude(violation="").exclude(violation="Not evaluated yet!") + # enc_vios_count = 0 + # for v in enc_vios: + # many = {} + # few = {} + # for i in range(10): + # many[i] = len(list(findall(f"{i} too many",v.violation))) + # few[i] = len(list(findall(f"{i} too few",v.violation))) + # enc_vios_count += i*(many[i] + few[i]) + + # # for a in s.split(":"): + # # if "too" in a: + # # print("\t",a) + # # ha_vios_count += int(a.split("too")[0].strip()) + # # quality_scen["HAWish"][prio] = [ha_vios_count,prioVal[prio] * ha_vios_count * gew["Home-/Away"]] + + + # quality_scen["EncWish"][prio] = [enc_vios_count,5*prioVal[prio] * enc_vios_count * gew["Encounters"]] + # total_vio += 5*prioVal[prio] * enc_vios_count * gew["Encounters"] + + # quality_scen["PAIR+HA+ENC"] = total_vio + + dynamic_rows = [] + + + """ PAIRINGS """ + pairings = Pairing.objects.filter(scenario=scenario,active=True) + pairing_tags = pairings.values_list('comment',flat=True).distinct() + + for tag in pairing_tags: + first_row.append("PAIRINGS") + row = f"{tag}" + dynamic_rows.append(row) + pairing_vios = pairings.filter(comment=tag).exclude(violation="") + pairing_viols_count = 0 + pairing_viols_obj = 0 + for v in pairing_vios: + pairing_viols_count += len(v.violation.split("__")) + pairing_viols_obj += 5*prioVal[v.prio] * len(v.violation.split("__")) + quality_scen[row] = [pairing_viols_count,pairing_viols_obj * gew["Pairings"]] + total_vio += pairing_viols_obj * gew["Pairings"] + + + """ HOME AWAY """ + tags = Tag.objects.filter(season=scenario.season,scheduler_hawishes__isnull=False).distinct() + + for tag in tags: + first_row.append("HOME AWAY") + row = f"{tag.name}" + dynamic_rows.append(row) + ha_vios = tag.scheduler_hawishes.filter(scenario=scenario,active=True).exclude(violation="") + ha_vios_count = 0 + ha_vios_obj = 0 + for v in ha_vios: + many = {} + few = {} + for i in range(10): + many[i] = len(list(findall(f"{i} too many",v.violation))) + few[i] = len(list(findall(f"{i} too few",v.violation))) + ha_vios_count += i*(many[i] + few[i]) + ha_vios_obj += i*(many[i] + few[i]) * prioVal[v.prio] + quality_scen[row] = [ha_vios_count, ha_vios_obj * gew["Home-/Away"]] + total_vio += ha_vios_obj * gew["Home-/Away"] + + + """ ENCOUNTERS """ + tags = Tag.objects.filter(season=scenario.season,scheduler_encwishes__isnull=False).distinct() + + for tag in tags: + first_row.append("ENCOUNTERS") + row = f"{tag.name}" + dynamic_rows.append(row) + # enc_vios = EncWish.objects.filter(scenario=scenario,prio=prio,active=True).exclude(violation="").exclude(violation="Not evaluated yet!") + enc_vios = tag.scheduler_encwishes.filter(scenario=scenario,active=True).exclude(violation="").exclude(violation="Not evaluated yet!") + enc_vios_count = 0 + enc_vios_obj = 0 + for v in enc_vios: + many = {} + few = {} + for i in range(10): + many[i] = len(list(findall(f"{i} too many",v.violation))) + few[i] = len(list(findall(f"{i} too few",v.violation))) + enc_vios_count += i*(many[i] + few[i]) + enc_vios_obj += i*(many[i] + few[i]) * prioVal[v.prio] * 5 + # for a in s.split(":"): + # if "too" in a: + # print("\t",a) + # ha_vios_count += int(a.split("too")[0].strip()) + # quality_scen["HAWish"][prio] = [ha_vios_count,prioVal[prio] * ha_vios_count * gew["Home-/Away"]] + quality_scen[row] = [enc_vios_count,enc_vios_obj * gew["Encounters"]] + total_vio += enc_vios_obj * gew["Encounters"] + + quality_scen["PAIR+HA+ENC"] = total_vio + + + + quality_dict[scenario.name] = quality_scen + + + writer.writerow(first_row[:5]+[""]+first_row[5:]+[""]+first_row[5:]) + rows = ["AScenario","Datetime","UCLPatterns","Overall Quality","PAIR+HA+ENC"] + rows += [""] + rows += [f"#({r})" for r in dynamic_rows] + rows += [""] + rows += [f"obj({r})" for r in dynamic_rows] + writer.writerow(rows) + + for scenario in quality_dict.keys(): + ll = [] + ll.append(f"{scenario}") + ll.append(f"{quality_dict[scenario]['Datetime'][0]}") + ll.append(f"{quality_dict[scenario]['UCLpatterns'][0]}") + ll.append(f"{quality_dict[scenario]['Overall Quality'][0]}") + ll.append(f"{quality_dict[scenario]['PAIR+HA+ENC']}") + ll.append("") + for row in dynamic_rows: + ll.append(f"{quality_dict[scenario][row][0]}") + ll.append("") + for row in dynamic_rows: + ll.append(f"{quality_dict[scenario][row][1]}") + writer.writerow(ll) + + + + + return response \ No newline at end of file diff --git a/scripts/check_data.py b/scripts/check_data.py new file mode 100644 index 0000000..94e39f8 --- /dev/null +++ b/scripts/check_data.py @@ -0,0 +1,85 @@ +# %% +PROJECT_PATH = '/home/md/Work/ligalytics/leagues_stable/' +import os, sys +sys.path.insert(0, PROJECT_PATH) +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "leagues.settings") +os.environ["DJANGO_ALLOW_ASYNC_UNSAFE"] = "true" + +from leagues import settings +# settings.DATABASES['default']['NAME'] = PROJECT_PATH+'/db.sqlite3' +settings.DATABASES['default']['ENGINE'] = 'django.db.backends.postgresql_psycopg2' +settings.DATABASES['default']['HOST'] = '0.0.0.0' +settings.DATABASES['default']['PORT'] = '5432' +settings.DATABASES['default']['USER'] = 'postgres' +settings.DATABASES['default']['PASSWORD'] = 'secret123' +settings.DATABASES['default']['NAME'] = 'mypgsqldb' +settings.DATABASES['default']['ATOMIC_REQUESTS'] = False +settings.DATABASES['default']['AUTOCOMMIT'] = True +settings.DATABASES['default']['CONN_MAX_AGE'] = 0 +settings.DATABASES['default']['CONN_HEALTH_CHECKS'] = False +settings.DATABASES['default']['OPTIONS'] = {} + +os.environ["XPRESSDIR"] = "/opt/xpressmp" +os.environ["XPRESS"] = "/opt/xpressmp/bin" +os.environ["LD_LIBRARY_PATH"] = os.environ["XPRESSDIR"] + "/lib" +os.environ["DYLD_LIBRARY_PATH"] = os.environ["XPRESSDIR"] + "/lib" +os.environ["SHLIB_PATH"] = os.environ["XPRESSDIR"] + "/lib" +os.environ["LIBPATH"] = os.environ["XPRESSDIR"] + "/lib" +os.environ["PYTHONPATH"] = os.environ["XPRESSDIR"] + "/lib" +os.environ["CLASSPATH"] = os.environ["XPRESSDIR"] + "/lib/xprs.jar" +os.environ["CLASSPATH"] = os.environ["XPRESSDIR"] + "/lib/xprb.jar" + os.pathsep + os.environ["CLASSPATH"] +os.environ["CLASSPATH"] = os.environ["XPRESSDIR"] + "/lib/xprm.jar" + os.pathsep + os.environ["CLASSPATH"] +os.environ["PATH"] = os.environ["XPRESSDIR"] + "/bin" + os.pathsep + os.environ["PATH"] + + +import django +django.setup() + +from scheduler.models import * +import pulp +from pulp import lpSum, value, XPRESS, GUROBI, PULP_CBC_CMD +from django.db.models import Q +from django.template.loader import render_to_string + +from qualifiers.models import * +from common.models import GlobalTeam, GlobalCountry +from scheduler.models import * + +from qualifiers.draws import groupTeams, optimize_inversions4 +from scheduler.solver.tasks.optimize import optimize +from scheduler.solver.tasks.optimize_submodels import ueluecl24_basicmodell_v2 + +from common.security import ESCAPE_WORDS + +import random +import time +import json +import csv +import networkx as nx +import matplotlib.pyplot as plt +from datetime import timedelta + +from django.contrib.sessions.models import Session + + + + +for word in ESCAPE_WORDS: + print("WORD",word) + for hawish in HAWish.objects.all(): + if hawish.reason.find(word) >= 0: + print(f"\t{hawish.scenario.season} found |{word}| in |{hawish.reason}|") + for encwish in EncWish.objects.all(): + if encwish.reason.find(word) >= 0: + print(f"\t{encwish.scenario.season} found |{word}| in |{encwish.reason}|") + for pairing in Pairing.objects.all(): + if pairing.comment.find(word) >= 0: + print(f"\t{pairing.scenario.season} found |{word}| in |{pairing.comment}|") + for team in Team.objects.all(): + if team.comments: + if team.comments.find(word) >= 0: + print(f"\t{team.season} found |{word}| in |{team.comments}|") + for blocking in Blocking.objects.all(): + if blocking.comment.find(word) >= 0: + print(f"\t{blocking.scenario.season} found |{word}| in |{blocking.comment}|") + diff --git a/solver/debug.py b/solver/debug.py new file mode 100755 index 0000000..4bafdbf --- /dev/null +++ b/solver/debug.py @@ -0,0 +1,78 @@ +# %% +PROJECT_PATH = '/home/md/Work/ligalytics/leagues_stable/' +import os, sys +sys.path.insert(0, PROJECT_PATH) +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "leagues.settings") +os.environ["DJANGO_ALLOW_ASYNC_UNSAFE"] = "true" + +from leagues import settings +# settings.DATABASES['default']['NAME'] = PROJECT_PATH+'/db.sqlite3' +settings.DATABASES['default']['ENGINE'] = 'django.db.backends.postgresql_psycopg2' +settings.DATABASES['default']['HOST'] = '0.0.0.0' +settings.DATABASES['default']['PORT'] = '5432' +settings.DATABASES['default']['USER'] = 'postgres' +settings.DATABASES['default']['PASSWORD'] = 'secret123' +settings.DATABASES['default']['NAME'] = 'mypgsqldb' +settings.DATABASES['default']['ATOMIC_REQUESTS'] = False +settings.DATABASES['default']['AUTOCOMMIT'] = True +settings.DATABASES['default']['CONN_MAX_AGE'] = 0 +settings.DATABASES['default']['CONN_HEALTH_CHECKS'] = False +settings.DATABASES['default']['OPTIONS'] = {} + +os.environ["XPRESSDIR"] = "/opt/xpressmp" +os.environ["XPRESS"] = "/opt/xpressmp/bin" +os.environ["LD_LIBRARY_PATH"] = os.environ["XPRESSDIR"] + "/lib" +os.environ["DYLD_LIBRARY_PATH"] = os.environ["XPRESSDIR"] + "/lib" +os.environ["SHLIB_PATH"] = os.environ["XPRESSDIR"] + "/lib" +os.environ["LIBPATH"] = os.environ["XPRESSDIR"] + "/lib" +os.environ["PYTHONPATH"] = os.environ["XPRESSDIR"] + "/lib" +os.environ["CLASSPATH"] = os.environ["XPRESSDIR"] + "/lib/xprs.jar" +os.environ["CLASSPATH"] = os.environ["XPRESSDIR"] + "/lib/xprb.jar" + os.pathsep + os.environ["CLASSPATH"] +os.environ["CLASSPATH"] = os.environ["XPRESSDIR"] + "/lib/xprm.jar" + os.pathsep + os.environ["CLASSPATH"] +os.environ["PATH"] = os.environ["XPRESSDIR"] + "/bin" + os.pathsep + os.environ["PATH"] + + +import django +django.setup() + +from scheduler.models import * +import pulp +from pulp import lpSum, value, XPRESS, GUROBI, PULP_CBC_CMD +from django.db.models import Q +from django.template.loader import render_to_string + +from qualifiers.models import * +from common.models import GlobalTeam, GlobalCountry +from scheduler.models import Season, Scenario, Team, DayObj, CountryClash, Country + +from qualifiers.draws import groupTeams, optimize_inversions4 +from scheduler.solver.tasks.optimize import optimize +from scheduler.solver.tasks.optimize_submodels import ueluecl24_basicmodell_v2 + +import random +import time +import json +import csv +import networkx as nx +import matplotlib.pyplot as plt +from datetime import timedelta + +from django.contrib.sessions.models import Session + + +scenario = Scenario.objects.get(id=9343) +# scenario = Scenario.objects.get(id=9368) +# scenario = Scenario.objects.get(id=1) +season = scenario.season + +s2 = scenario.id +user_name = 'md' +user_is_staff = True +runMode = 'Improve' +localsearch_time = 0 +RUN_ENV = 'local' +SOLVER = 'xpress' + + +sol = optimize(task=None, s2=s2, user_name=user_name, user_is_staff=user_is_staff, + runMode=runMode, localsearch_time=localsearch_time, RUN_ENV=RUN_ENV, solver=SOLVER) diff --git a/uefa/cycle24/research/script.py b/uefa/cycle24/research/script.py new file mode 100755 index 0000000..06069d9 --- /dev/null +++ b/uefa/cycle24/research/script.py @@ -0,0 +1,114 @@ +# %% +PROJECT_PATH = '/home/md/Work/ligalytics/leagues_stable/' +import os, sys +sys.path.insert(0, PROJECT_PATH) +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "leagues.settings") +os.environ["DJANGO_ALLOW_ASYNC_UNSAFE"] = "true" + +from leagues import settings +# settings.DATABASES['default']['NAME'] = PROJECT_PATH+'/db.sqlite3' +settings.DATABASES['default']['ENGINE'] = 'django.db.backends.postgresql_psycopg2' +settings.DATABASES['default']['HOST'] = '0.0.0.0' +settings.DATABASES['default']['PORT'] = '5432' +settings.DATABASES['default']['USER'] = 'postgres' +settings.DATABASES['default']['PASSWORD'] = 'secret123' +settings.DATABASES['default']['NAME'] = 'mypgsqldb' +settings.DATABASES['default']['ATOMIC_REQUESTS'] = False +settings.DATABASES['default']['AUTOCOMMIT'] = True +settings.DATABASES['default']['CONN_MAX_AGE'] = 0 +settings.DATABASES['default']['CONN_HEALTH_CHECKS'] = False +settings.DATABASES['default']['OPTIONS'] = {} + +os.environ["XPRESSDIR"] = "/opt/xpressmp" +os.environ["XPRESS"] = "/opt/xpressmp/bin" +os.environ["LD_LIBRARY_PATH"] = os.environ["XPRESSDIR"] + "/lib" +os.environ["DYLD_LIBRARY_PATH"] = os.environ["XPRESSDIR"] + "/lib" +os.environ["SHLIB_PATH"] = os.environ["XPRESSDIR"] + "/lib" +os.environ["LIBPATH"] = os.environ["XPRESSDIR"] + "/lib" +os.environ["PYTHONPATH"] = os.environ["XPRESSDIR"] + "/lib" +os.environ["CLASSPATH"] = os.environ["XPRESSDIR"] + "/lib/xprs.jar" +os.environ["CLASSPATH"] = os.environ["XPRESSDIR"] + "/lib/xprb.jar" + os.pathsep + os.environ["CLASSPATH"] +os.environ["CLASSPATH"] = os.environ["XPRESSDIR"] + "/lib/xprm.jar" + os.pathsep + os.environ["CLASSPATH"] +os.environ["PATH"] = os.environ["XPRESSDIR"] + "/bin" + os.pathsep + os.environ["PATH"] + + +import django +django.setup() + +from scheduler.models import * +import pulp +from pulp import lpSum, value, XPRESS, GUROBI, PULP_CBC_CMD +from django.db.models import Q +from django.template.loader import render_to_string + +from scheduler.models import * + + +import random +import time +import json +import csv +import networkx as nx +import matplotlib.pyplot as plt +from datetime import timedelta + +from django.contrib.sessions.models import Session +from pprint import pprint + +ucl_basescenario = Scenario.objects.get(id=9543) +# scenario = Scenario.objects.get(id=9368) +# scenario = Scenario.objects.get(id=1) +season = ucl_basescenario.season + + +days = Day.objects.filter(season=season) +teams = Team.objects.filter(season=season) + +getTeamById = {team.id:team for team in teams} +getDayById = {day.id:day for day in days} + +user_name = 'md' +user_is_staff = True +runMode = 'New' +localsearch_time = 0 +RUN_ENV = 'local' +SOLVER = 'xpress' + + +solverstats = json.loads(ucl_basescenario.solverstats_json) + +solverstats.keys() + +# home_in_round = { +# (r,t):0 for t in teams for r in range(1,season.nRounds+1) +# } + + +# away_in_round = { +# (r,t):0 for t in teams for r in range(1,season.nRounds+1) +# } + +# scenarios = season.scenarios.exclude(id=ucl_basescenario.id) + +# for scenario in scenarios: + +# solverstats = json.loads(scenario.solverstats_json) + +# if solverstats.get('info-Improve'): +# print(solverstats['info-Improve']) +# break + + +# # solution = scenario.sol_solution() + +# for game in solverstats['premodell']['partial_solution']: +# home_in_round[(getDayById[game['days'][0]].round,getTeamById[game['home'][0]])] += 1 +# away_in_round[(getDayById[game['days'][0]].round,getTeamById[game['away'][0]])] += 1 +# # sol = optimize(task=None, s2=ucl_basescenario.id, user_name=user_name, user_is_staff=user_is_staff, +# # runMode=runMode, localsearch_time=localsearch_time, RUN_ENV=RUN_ENV, solver=SOLVER) +# home_in_round = sorted(home_in_round.items(), key=lambda x: x[0][0]) +# pprint(home_in_round) +# away_in_round = sorted(away_in_round.items(), key=lambda x: x[0][0]) +# pprint(away_in_round) + +# %% diff --git a/uefa/feb24_checkandschedule/uefa_tester_24.py b/uefa/feb24_checkandschedule/uefa_tester_24.py new file mode 100644 index 0000000..71788db --- /dev/null +++ b/uefa/feb24_checkandschedule/uefa_tester_24.py @@ -0,0 +1,285 @@ +import csv +import operator +import os +import random +import sys +import time +import timeit +import datetime +# import networkx as nx + +from datetime import datetime + +from dateutil.parser import parse +from math import sqrt, pow, sin, cos, atan2, pi + +try: + import simplejson as json +except: + import json + +from gurobipy import * + +# import config + +getTeamsOfPot = { p: [ p+str(i) for i in range(1,10) ] for p in ["A","B","C","D"] } + +allTeams = getTeamsOfPot['A']+getTeamsOfPot['B']+getTeamsOfPot['C']+getTeamsOfPot['D'] +activeTeams= allTeams +pots = getTeamsOfPot.keys() +# print (pots) + +gamesAgainst = { (p1,p2) : 2 for p1 in pots for p2 in pots} + +# for p in pots: +# gamesAgainst[(p,p)]=2 + +# for (p1,p2) in [('A','D'), ('D','A'), ('B','C'), ('C','B') ]: +# gamesAgainst[(p1,p2)]=2 + +nMatchDays=8 + + +# for p1 in pots: +# for p2 in pots: +# print (p1,p2,gamesAgainst[(p1,p2)]) +# print() + +# print (allTeams) + +cntr=0 + +mbus = Model("Draw") + +possAssigns= [ (t1,t2) for t1 in allTeams for t2 in allTeams if t1!=t2] +# print (possAssigns) + + +plays = mbus.addVars(possAssigns, vtype=GRB.BINARY) + +for (t1,t2) in possAssigns: + if t11: + nSims=int(sys.argv[1]) + + +solutions =[] +solution_strings =[] +for i in range(nSims): + if i%10==0: + print(i) + cntr+=1 + mbus.setObjective(quicksum([ int(100*random.random()) * plays[pa] for pa in possAssigns ])) + # mbus.setObjective(quicksum([ max(0,int(pa[0][1]) - int(pa[1][1] )) * plays[pa] for pa in possAssigns ])) + + # print ("DRAWING !!!!") + + mbus.optimize() + sol= [ pa for pa in possAssigns if plays[pa].x>0.9] + solutions.append(sol.copy()) + + if debug: + print ("Solution") + # exit(0) + print ("INPOT") + for p in pots: + print ("") + print ("POT",p) + for (t1,t2) in [(t1,t2) for (t1,t2) in sol if t1= int(0.5*gamesAgainst[(p,p2)]) ) + else: + for (t1,t2) in sol : + if t10.9 ] + # print( cal_sol) + # for pa in cal_sol : + # print (pa) + # # print (cal.status) + # print(cal.objVal) + + if cal.status!=2 or cal.objVal>-288: + sol= [ (t1,t2) for (t1,t2) in sol if t10.9] + # print (pas2_sol) + + + showSolution= False + showSolution= True + if showSolution and debug: + for md in mds: + print (md,":") + for (t1,t2,md2) in pas2_sol: + # if md2==md and t11 and showSolution: + newGm = [ gg for gg in solutions[-1] if gg not in solutions[-2]] + + # print (oldGm) + # print (newGm) +# print (solution_strings) + +print (len(set(solution_strings)) ) + +f = open("draw_feasibility/log_"+datetime.now().strftime("%Y%m%d_%H%M%S")+"_"+str(nSims)+".txt", "a") +for s in solutions: + for s2 in s: + f.write(s2[0]+s2[1]) + f.write("\n") +f.close() + +print ("done" , datetime.now()-now ) + diff --git a/uefa/scripts/solve_uefa24.py b/uefa/scripts/solve_uefa24.py index 187d945..ffb23be 100755 --- a/uefa/scripts/solve_uefa24.py +++ b/uefa/scripts/solve_uefa24.py @@ -43,11 +43,12 @@ from django.template.loader import render_to_string from qualifiers.models import * from common.models import GlobalTeam, GlobalCountry -from scheduler.models import Season, Scenario, Team, DayObj, CountryClash, Country +from scheduler.models import Season, Scenario, Team, DayObj, CountryClash, Country, Day from qualifiers.draws import groupTeams, optimize_inversions4 from scheduler.solver.tasks.optimize import optimize from scheduler.solver.tasks.optimize_submodels import ueluecl24_basicmodell_v2 +from scheduler.solver.optimizer import optimize_2phases import random import time @@ -60,114 +61,49 @@ from datetime import timedelta from django.contrib.sessions.models import Session -scenario = Scenario.objects.get(id=9051) -# scenario = Scenario.objects.get(id=9368) -# scenario = Scenario.objects.get(id=1) -season = scenario.season +# scenario = Scenario.objects.get(id=9534) +# scenario = Scenario.objects.get(id=9379) +# scenario = Scenario.objects.get(id=9361) +scenario = Scenario.objects.get(id=9579) +season = scenario.season s2 = scenario.id user_name = 'md' user_is_staff = True -runMode = 'New' -localsearch_time = 0 +runMode = 'Improve' +localsearch_time = 30 RUN_ENV = 'local' SOLVER = 'xpress' -# Conference.objects.filter(scenario=scenario,name__in=["UEL","UECL"]).update(reopt=True) +nRounds = season.nRounds +rounds=range(1,nRounds+1) +dayObjects = Day.objects.filter(season=season).values() +getDays = { r : [] for r in rounds} +for d in dayObjects: + if d['round']>0 : + getDays[d['round']].append(d['id']) + + sol = optimize(task=None, s2=s2, user_name=user_name, user_is_staff=user_is_staff, runMode=runMode, localsearch_time=localsearch_time, RUN_ENV=RUN_ENV, solver=SOLVER) +# sol = optimize_2phases(s2, user_name, user_is_staff, runMode=runMode, localsearch_time=localsearch_time, RUN_ENV=RUN_ENV, SOLVER=SOLVER) +# partial_solution = [] +# res_objective, res_games = ueluecl24_basicmodell_v2(SOLVER, s2, partial_solution) + +# print(res_games) + +# res_games = [(42255, 42321, 1), (42251, 42252, 1), (42244, 42317, 1), (42291, 42288, 1), (42290, 42310, 1), (42289, 42327, 1), (42277, 42282, 1), (42326, 42312, 1), (42325, 42314, 1), (42324, 42280, 1), (42319, 42320, 1), (42316, 42322, 1), (42315, 42273, 1), (42311, 42296, 1), (42313, 42305, 1), (42318, 42299, 1), (42323, 42329, 1), (42328, 42304, 1), (42330, 42336, 2), (42333, 42343, 2), (42250, 42293, 2), (42273, 42290, 2), (42271, 42300, 2), (42253, 42266, 2), (42252, 42311, 2), (42305, 42326, 2), (42304, 42319, 2), (42299, 42289, 2), (42297, 42332, 2), (42296, 42323, 2), (42288, 42277, 2), (42282, 42313, 2), (42280, 42315, 2), (42279, 42348, 2), (42278, 42286, 2), (42346, 42345, 2), (42342, 42254, 2), (42340, 42261, 2), (42339, 42247, 2), (42338, 42331, 2), (42335, 42344, 2), (42329, 42324, 2), (42327, 42328, 2), (42322, 42318, 2), (42321, 42251, 2), (42320, 42316, 2), (42317, 42291, 2), (42314, 42255, 2), (42312, 42244, 2), (42310, 42325, 2), (42307, 42337, 2), (42347, 42349, 2), (42350, 42334, 2), (42267, 42341, 2), (42293, 42307, 3), (42273, 42304, 3), (42266, 42342, 3), (42261, 42267, 3), (42254, 42338, 3), (42247, 42350, 3), (42305, 42312, 3), (42300, 42297, 3), (42299, 42244, 3), (42296, 42315, 3), (42291, 42280, 3), (42290, 42325, 3), (42286, 42333, 3), (42282, 42318, 3), (42277, 42313, 3), (42349, 42339, 3), (42345, 42340, 3), (42344, 42278, 3), (42341, 42346, 3), (42336, 42335, 3), (42348, 42330, 3), (42334, 42253, 3), (42331, 42250, 3), (42329, 42252, 3), (42327, 42317, 3), (42324, 42326, 3), (42322, 42251, 3), (42321, 42314, 3), (42320, 42288, 3), (42316, 42255, 3), (42311, 42319, 3), (42310, 42289, 3), (42328, 42323, 3), (42332, 42271, 3), (42337, 42279, 3), (42343, 42347, 3), (42255, 42311, 4), (42250, 42350, 4), (42271, 42293, 4), (42266, 42333, 4), (42254, 42348, 4), (42253, 42339, 4), (42252, 42305, 4), (42251, 42310, 4), (42244, 42282, 4), (42304, 42290, 4), (42300, 42349, 4), (42289, 42321, 4), (42286, 42340, 4), (42280, 42329, 4), (42277, 42322, 4), (42346, 42336, 4), (42341, 42337, 4), (42342, 42344, 4), (42338, 42297, 4), (42335, 42261, 4), (42334, 42345, 4), (42331, 42330, 4), (42326, 42316, 4), (42325, 42291, 4), (42319, 42273, 4), (42317, 42328, 4), (42315, 42299, 4), (42314, 42327, 4), (42312, 42324, 4), (42313, 42320, 4), (42318, 42296, 4), (42323, 42288, 4), (42332, 42247, 4), (42343, 42279, 4), (42347, 42307, 4), (42267, 42278, 4), (42330, 42254, 5), (42333, 42334, 5), (42293, 42347, 5), (42273, 42312, 5), (42261, 42332, 5), (42247, 42300, 5), (42244, 42315, 5), (42304, 42316, 5), (42299, 42322, 5), (42297, 42250, 5), (42289, 42305, 5), (42288, 42325, 5), (42282, 42327, 5), (42280, 42251, 5), (42279, 42267, 5), (42278, 42335, 5), (42349, 42271, 5), (42345, 42338, 5), (42344, 42253, 5), (42340, 42341, 5), (42339, 42343, 5), (42336, 42342, 5), (42348, 42266, 5), (42326, 42290, 5), (42324, 42255, 5), (42321, 42252, 5), (42320, 42329, 5), (42319, 42310, 5), (42317, 42311, 5), (42307, 42331, 5), (42313, 42314, 5), (42318, 42291, 5), (42323, 42277, 5), (42328, 42296, 5), (42337, 42286, 5), (42350, 42346, 5), (42330, 42342, 6), (42255, 42289, 6), (42250, 42300, 6), (42261, 42346, 6), (42252, 42319, 6), (42251, 42323, 6), (42305, 42244, 6), (42297, 42271, 6), (42296, 42280, 6), (42291, 42324, 6), (42290, 42282, 6), (42288, 42304, 6), (42286, 42293, 6), (42279, 42334, 6), (42278, 42347, 6), (42349, 42332, 6), (42341, 42254, 6), (42340, 42348, 6), (42339, 42338, 6), (42336, 42333, 6), (42335, 42253, 6), (42331, 42266, 6), (42329, 42273, 6), (42327, 42321, 6), (42325, 42320, 6), (42322, 42313, 6), (42316, 42318, 6), (42315, 42328, 6), (42314, 42317, 6), (42312, 42277, 6), (42311, 42326, 6), (42310, 42299, 6), (42307, 42344, 6), (42343, 42247, 6), (42350, 42337, 6), (42267, 42345, 6), (42333, 42339, 7), (42293, 42336, 7), (42271, 42343, 7), (42266, 42297, 7), (42254, 42261, 7), (42253, 42330, 7), (42247, 42279, 7), (42300, 42307, 7), (42346, 42286, 7), (42345, 42331, 7), (42344, 42350, 7), (42342, 42250, 7), (42338, 42349, 7), (42348, 42267, 7), (42334, 42278, 7), (42332, 42341, 7), (42337, 42340, 7), (42347, 42335, 7), (42255, 42325, 7), (42273, 42318, 7), (42305, 42304, 7), (42296, 42288, 7), (42280, 42320, 7), (42277, 42310, 7), (42326, 42319, 7), (42324, 42244, 7), (42322, 42327, 7), (42321, 42290, 7), (42317, 42282, 7), (42315, 42291, 7), (42314, 42252, 7), (42312, 42316, 7), (42311, 42251, 7), (42313, 42329, 7), (42323, 42289, 7), (42328, 42299, 7), (42252, 42328, 8), (42251, 42273, 8), (42244, 42323, 8), (42304, 42313, 8), (42299, 42255, 8), (42291, 42314, 8), (42290, 42311, 8), (42289, 42277, 8), (42288, 42321, 8), (42282, 42280, 8), (42329, 42305, 8), (42327, 42315, 8), (42325, 42322, 8), (42320, 42312, 8), (42319, 42324, 8), (42316, 42317, 8), (42310, 42296, 8), (42318, 42326, 8)] -# ueluecl24_basicmodell_v2(SOLVER,scenario,[],[],maxTime=3600) +# partial_solution = [] +# for (t1,t2,r) in res_games: +# partial_solution.append({"home": [t1], "away": [t2], "days": getDays[r], "time": None , "changeable": False }) -# %% -# hawishes = HAWish.objects.filter(scenario=scenario) - -# # elemHaWishes ={e['id'] : [] for e in hawishes} -# # elemHaWishTeams={} -# # elemHaWishDays ={} -# # elemHaWishFirstDay ={} -# # elemHaWishNum ={} - -# cntr =1 -# for e in hawishes: -# elemTeams = [[t] for t in e.get_teams()] if e.forEachTeam else [e.get_teams()] -# elemRounds = [[d] for d in e.get_rounds()] if e.forEachDay else [e.get_rounds()] - -# if e.homeAway == "HOME": -# if e.minGames > 0: -# for teamset in elemTeams: -# for roundset in elemRounds: -# model += lpSum([x[r,t.id,t2.id] for t in teamset for r in roundset for t2 in teams if (r,t.id,t2.id) in x.keys()]) >= e.minGames - seedViolation[seed.id] +# res_objective, res_games = ueluecl24_basicmodell_v2(SOLVER, s2, partial_solution) - - - -# # elemHaWishes[e['id']]=[] -# # allElemDays=[] -# # thisDaySet=[] -# # lastDaySet = [] -# # # print("elemDays",elemDays) -# # for d in elemDays : -# # # print (e) -# # if (e['forEachDay'] or e['forOneDay']) and e['timeframe']!=1: -# # thisDaySet=[] -# # # day1= parse(getDayById[d[0]]['day']) -# # day1= getDateTimeDay[d[0]] -# # if e['timeframe']>1: -# # day2=day1 + datetime.timedelta(days=e['timeframe']-1) -# # for d3 in days: -# # # dt = parse(getDayById[d3]['day']) -# # dt= getDateTimeDay[d3] -# # if day1<=dt and dt<=day2 : -# # thisDaySet.append(d3) -# # else: -# # r1 = getDayById[d[0]]['round'] -# # for d3 in days: -# # dt= getDateTimeDay[d3] -# # # dt = parse(getDayById[d3]['day']) -# # if day1<=dt and r1 <= getRoundByDay[d3] and getRoundByDay[d3]< r1 + (-e['timeframe']) : -# # thisDaySet.append(d3) -# # # print (" ROUND HA WISH ", e['reason'], thisDaySet, e['timeframe']) -# # else: -# # thisDaySet=d - -# # # only create wish id new day set is superset -# # if len([d for d in thisDaySet if d not in lastDaySet])>0: -# # for t in elemTeams: -# # cntr+=1 -# # elemHaWishes[e['id']].append(cntr) -# # elemHaWishTeams[cntr]=t -# # elemHaWishDays[cntr]=thisDaySet.copy() -# # elemHaWishFirstDay[cntr]=d[0] -# # elemHaWishNum[cntr]=1 -# # if e['maxGames'] ==0: -# # elemHaWishNum[cntr]=len(elemHaWishDays[cntr])*len(elemHaWishTeams[cntr]) -# # if e['minGames'] > 0: -# # elemHaWishNum[cntr]=e['minGames'] -# # lastDaySet = thisDaySet.copy() -# # allElemDays+= thisDaySet - -# # hawRounds[e['id']]=[] -# # for d3 in set(allElemDays): -# # hawRounds[e['id']]+=getRoundsByDay[d3] -# # hawRounds[e['id']]=sorted(list(set(hawRounds[e['id']]))) -# # hawRoundsString[e['id']]= "" -# # for r in hawRounds[e['id']]: -# # hawRoundsString[e['id']]+= str(r)+"_" -# # if hawRoundsString[e['id']]!="": -# # hawRoundsString[e['id']]=hawRoundsString[e['id']][:-1] -# # e['nWishes'] = sum ( elemHaWishNum[ee] for ee in elemHaWishes[e['id']] ) - -# # if e['forOneDay']: -# # e['nWishes'] = e['forOneDayNum'] * len(elemTeams) - -# # nElemHaWishes= sum(w['nWishes'] for w in hawishes) - -# # %% diff --git a/uefa/scripts/uefa24_simulator.py b/uefa/scripts/uefa24_simulator.py index 1fb3b48..39adcb0 100755 --- a/uefa/scripts/uefa24_simulator.py +++ b/uefa/scripts/uefa24_simulator.py @@ -108,14 +108,14 @@ def getVal(v): # basescenario = Scenario.objects.get(id=9306) -basescenario = Scenario.objects.get(id=9279) # scenario = Scenario.objects.get(id=9308) +basescenario = Scenario.objects.get(id=9414) s2 = basescenario.id user_name = 'md' user_is_staff = True runMode = 'New' -localsearch_time = 60 +localsearch_time = 0 RUN_ENV = 'local' SOLVER = 'xpress' diff --git a/uefa/uefa_api/api.py b/uefa/uefa_api/api.py new file mode 100644 index 0000000..0ccb90a --- /dev/null +++ b/uefa/uefa_api/api.py @@ -0,0 +1,22 @@ +# %% +import requests + + +prod_url = 'https://uefadigitalapi.developer.azure-api.net' +prod_primary_key = '7dfa861240aa40f8a834990c24f1a66d' +prod_secondary_key = '4451dcc1ad4f41b2aa6af96cc5a1256a' +pre_url = 'https://uefadigitalapipre.developer.azure-api.net' +pre_primary_key = '1decf93425944f8b9e6dc7226a3b8477' +pre_secondary_key = '14771f5c67b74836a59f777cb543cc0f' + + + +r=requests.get("https://api.digital.uefa.com/comp/v2/competitions/1/seasons", headers={"Cache-Control":"no-cache","Ocp-Apim-Subscription-Key":"7dfa861240aa40f8a834990c24f1a66d"}) +# %% +r.json() +# %% + +r=requests.get("https://api.pre.digital.uefa.com/comp/v2/competitions/1/seasons", headers={"Cache-Control":"no-cache","Ocp-Apim-Subscription-Key":"14771f5c67b74836a59f777cb543cc0f"}) +# %% +r.json() +# %%