Skip to content

Commit 11c61d4

Browse files
author
Ali Raza
committed
commit
1 parent 65b2d6a commit 11c61d4

File tree

4 files changed

+637
-0
lines changed

4 files changed

+637
-0
lines changed

‎CloudController/client.py

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
#!/usr/bin/env python
2+
3+
import socket
4+
5+
TCP_IP = '10.224.41.29'
6+
TCP_PORT = 60002
7+
BUFFER_SIZE = 1024
8+
MESSAGE = "Hello, World!"
9+
10+
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
11+
s.connect((TCP_IP, TCP_PORT))
12+
s.send(MESSAGE)
13+
data = s.recv(BUFFER_SIZE)
14+
s.close()
15+
16+
print "received data:", data
17+
18+

‎CloudController/preFetching.old.py

Lines changed: 298 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,298 @@
1+
import time
2+
import thread
3+
import copy
4+
from pyvirtualdisplay import Display
5+
from selenium import webdriver
6+
from selenium.webdriver.common.proxy import *
7+
import selenium.webdriver.firefox.webdriver as fwb
8+
import socket as dummysocket
9+
import cPickle
10+
import Queue as Q
11+
import logging
12+
import controller
13+
import constants
14+
global ALL_WEBSITES, PREFETCHING_QUEUE, PREFETCHING_LIST
15+
16+
17+
18+
logger_1 = logging.getLogger('simple_logger')
19+
logger_1.setLevel(logging.INFO)
20+
hdlr_1 = logging.FileHandler('prefetching.log')
21+
logger_1.addHandler(hdlr_1)
22+
23+
logger_2 = logging.getLogger('simple_logger')
24+
logger_2.setLevel(logging.INFO)
25+
hdlr_2 = logging.FileHandler('utilitycalculation.log')
26+
logger_2.addHandler(hdlr_2)
27+
28+
29+
PREFETCHING_LIST = []
30+
PREFETCHING_QUEUE = Q.PriorityQueue()
31+
32+
33+
MAX_BOOTSTRAP = constants.MAX_BOOTSTRAP
34+
BOOTSTRAPSITES = {}
35+
BOOTSTRAPSITES_COUNTER = 0
36+
PREFETCHING_COUNTER = 0
37+
ALL_WEBSITES = {}
38+
TIME = constants.INTERVAL_PREFETCHING
39+
40+
41+
def startPrefetching(num):
42+
thread.start_new_thread(receiveLogs, (1,))
43+
thread.start_new_thread(bootstrap, (1,))
44+
thread.start_new_thread(sitesPrefetching, (1,))
45+
46+
47+
48+
def openPage (webpage, check):
49+
50+
global BOOTSTRAPSITES_COUNTER, PREFETCHING_COUNTER
51+
try:
52+
display = Display(visible=0, size=(1920,1080))
53+
display.start()
54+
myProxy = "127.0.0.1:9999"
55+
proxy = Proxy ({
56+
'proxyType':ProxyType.MANUAL,
57+
'httpProxy': myProxy,
58+
'ftpProxy': myProxy,
59+
'sslProxy': '',
60+
'noProxy': ''
61+
})
62+
63+
binary = fwb.FirefoxBinary(firefox_path='/usr/bin/firefox')
64+
profile = webdriver.FirefoxProfile()
65+
profile.set_preference('datareporting.healthreport.uploadEnabled', False)
66+
profile.set_preference('datareporting.healthreport.service.enabled', False)
67+
profile.set_preference('datareporting.healthreport.service.firstRun', False)
68+
profile.set_preference('app.update.auto', False)
69+
profile.set_preference('app.update.enabled', False)
70+
profile.set_preference('browser.cache.disk.enable', False)
71+
profile.set_preference('browser.cache.memory.enable', False)
72+
profile.set_preference('browser.cache.offline.enable', False)
73+
profile.set_preference('network.http.use-cache', False)
74+
profile.set_preference('network.dns.disablePrefetch', True)
75+
profile.set_preference('network.http.accept-encoding', '')
76+
77+
#for modifying header
78+
profile.add_extension( 'modify_headers-0.7.1.1-fx.xpi')
79+
profile.set_preference('modifyheaders.headers.count', 1)
80+
profile.set_preference('modifyheaders.headers.action0', "Add")
81+
profile.set_preference('modifyheaders.headers.name0', 'webpage')
82+
profile.set_preference('modifyheaders.headers.value0', check+':'+webpage)
83+
profile.set_preference('modifyheaders.headers.enabled0', True)
84+
profile.set_preference('modifyheaders.config.active', True)
85+
profile.set_preference('modifyheaders.config.alwaysOn', True)
86+
#
87+
profile.update_preferences()
88+
browser = webdriver.Firefox(firefox_profile=profile, firefox_binary=binary, proxy=proxy)
89+
browser.set_page_load_timeout(70)
90+
browser.set_window_size(1920, 1080)
91+
92+
try:
93+
browser.get(webpage)
94+
print "-- Finished loading ", browser.title
95+
except:
96+
print "-- Problem loading "
97+
display.stop()
98+
del profile
99+
browser.quit()
100+
del browser
101+
102+
if check =='b':
103+
BOOTSTRAPSITES_COUNTER -= 1
104+
else:
105+
PREFETCHING_COUNTER -= 1
106+
return
107+
108+
109+
del profile
110+
browser.quit()
111+
del browser
112+
113+
if check =='b':
114+
BOOTSTRAPSITES_COUNTER -= 1
115+
else:
116+
PREFETCHING_COUNTER -= 1
117+
118+
display.stop()
119+
120+
except:
121+
122+
if check =='b':
123+
BOOTSTRAPSITES_COUNTER -= 1
124+
else:
125+
PREFETCHING_COUNTER -= 1
126+
return
127+
128+
129+
130+
def bootstrap(a):
131+
132+
global BOOTSTRAPSITES, BOOTSTRAPSITES_COUNTER
133+
print "Bootstraping Thread ...\n"
134+
135+
while True:
136+
137+
if len (BOOTSTRAPSITES ) > 0:
138+
139+
for item in BOOTSTRAPSITES.keys():
140+
141+
if BOOTSTRAPSITES_COUNTER < 6:
142+
143+
if BOOTSTRAPSITES [item][1] <= time.time():
144+
print ('BOOTSTRAP: ', item, 'FOR: ',BOOTSTRAPSITES[item][0] )
145+
log_string = 'BOOTSTRAP: '+str(time.time()) +' :'+item
146+
logger_1.info(log_string)
147+
BOOTSTRAPSITES_COUNTER +=1
148+
149+
try:
150+
thread.start_new_thread( openPage, (item, 'b', ))
151+
except:
152+
pass
153+
154+
BOOTSTRAPSITES [item][0]-=1
155+
BOOTSTRAPSITES [item][1]=time.time()+constants.INTERVAL_BOOTSTRAP
156+
print BOOTSTRAPSITES [item][1]
157+
158+
if BOOTSTRAPSITES [item][0] <=0 :
159+
print 'ADDES TO PREFETCHING_LIST'
160+
log_string = 'BOOTSTRAP: ADDED_TO_PREFETCHING_LIST: '+item
161+
logger_1.info(log_string)
162+
PREFETCHING_LIST.append(item)
163+
del BOOTSTRAPSITES[item]
164+
165+
while BOOTSTRAPSITES_COUNTER >=5:
166+
time.sleep(1)
167+
168+
169+
170+
171+
172+
173+
174+
def sitesPrefetching (number):
175+
PREFETCHING_BOOL = False
176+
177+
while True:
178+
179+
global PREFETCHING_QUEUE , TIME, PREFETCHING_LIST, PREFETCHING_COUNTER
180+
181+
if len(PREFETCHING_LIST) > 0:
182+
PREFETCHING_QUEUE = calculateUtilities()
183+
184+
startTime = time.time()
185+
while not PREFETCHING_QUEUE.empty():
186+
PREFETCHING_BOOL = True
187+
w = PREFETCHING_QUEUE.get()
188+
log_string = 'PREFETCHING: '+str(time.time()) +' :'+w[1]
189+
print log_string
190+
logger_1.info(log_string)
191+
try:
192+
PREFETCHING_COUNTER +=1
193+
thread.start_new_thread(openPage, (w[1] , 'p', ))
194+
except:
195+
pass
196+
197+
while PREFETCHING_COUNTER >=5:
198+
time.sleep(1)
199+
200+
if time.time() - startTime >= TIME:
201+
break # we have reached the end of the slot for prefetching we should stop and start the next slot
202+
203+
if PREFETCHING_BOOL:
204+
if time.time() - startTime < TIME:
205+
print 'Going to sleep:' + str(TIME) +' '+str(time.time() - startTime)
206+
time.sleep(TIME - (time.time() - startTime))
207+
208+
209+
210+
def receiveLogs(num):
211+
212+
global ALL_WEBSITES
213+
CONTROLLER_IP = constants.CONTROLLER_IP
214+
CONTROLLER_PORT = constants.CONTROLLER_PORT_LOGS
215+
s = dummysocket.socket(dummysocket.AF_INET, dummysocket.SOCK_STREAM)
216+
s.setsockopt(dummysocket.SOL_SOCKET, dummysocket.SO_REUSEADDR, 1)
217+
s.bind((CONTROLLER_IP, CONTROLLER_PORT))
218+
219+
while 1:
220+
try:
221+
s.listen(1)
222+
conn, addr = s.accept()
223+
MESSAGE= ""
224+
data = conn.recv(1024)
225+
while data:
226+
MESSAGE += data
227+
data = conn.recv(1024)
228+
print 'Got Logs'
229+
websites = cPickle.loads(MESSAGE)
230+
tmp = websites
231+
print tmp
232+
for siteInfo in tmp:
233+
if siteInfo[0] in ALL_WEBSITES:
234+
ALL_WEBSITES[siteInfo[0]].N = 0.7*ALL_WEBSITES[siteInfo[0]].N + 0.3*siteInfo[1] # TODO: fix the ewma alpha parameter (at the moment random number is given)
235+
else:
236+
BOOTSTRAPSITES [siteInfo[0]]=[MAX_BOOTSTRAP , 0]
237+
ALL_WEBSITES[siteInfo[0]]=controller.WebPage(siteInfo[1])
238+
log_string = 'ADDED FROM LOGS: '+siteInfo[0]
239+
logger_1.info(log_string)
240+
except:
241+
s = dummysocket.socket(dummysocket.AF_INET, dummysocket.SOCK_STREAM)
242+
s.setsockopt(dummysocket.SOL_SOCKET, dummysocket.SO_REUSEADDR, 1)
243+
s.bind((CONTROLLER_IP, CONTROLLER_PORT))
244+
pass
245+
return
246+
247+
248+
# tmp = [ ('http://www.cnn.com/', 10), ('http://www.bbc.com/', 10), ('http://www.yahoo.com/', 10), ('http://www.apple.com/', 10), ('http://www.souq.com/', 10),('http://www.faiza.com/', 10), ('http://www.ebay.com/', 10) ]
249+
250+
# for siteInfo in tmp:
251+
# if siteInfo[0] in ALL_WEBSITES:
252+
# ALL_WEBSITES[siteInfo[0]].N = 0.7*ALL_WEBSITES[siteInfo[0]].N + 0.3*siteInfo[1] # TODO: fix the ewma alpha parameter (at the moment random number is given)
253+
# else:
254+
# BOOTSTRAPSITES [siteInfo[0]]=[MAX_BOOTSTRAP , 0]
255+
# ALL_WEBSITES[siteInfo[0]]=controller.WebPage(siteInfo[1])
256+
# log_string = 'ADDED FROM LOGS: '+siteInfo[0]
257+
# logger_1.info(log_string)
258+
259+
260+
261+
262+
263+
264+
def calculateUtilities():
265+
global ALL_WEBSITES, PREFETCHING_QUEUE, PREFETCHING_LIST
266+
PREFETCHING_QUEUE = Q.PriorityQueue()
267+
for webpage in PREFETCHING_LIST:
268+
print 'Calculating-Utilities ...'
269+
n_t = float(0.000)
270+
d_t = float(0.000)
271+
n_b = float(0.000)
272+
d_b = float(0.000)
273+
webPageObjects = ALL_WEBSITES[webpage].objects
274+
logger_2.info('Utility Calculation: ' + webpage)
275+
for o in webPageObjects.keys():
276+
x1, x2, x3, x4, st = webPageObjects[o].calculateUtilities()
277+
n_t = n_t + x1
278+
d_t = d_t + x2
279+
n_b = n_b + x3
280+
d_b = d_b + x4
281+
logger_2.info(st)
282+
283+
if d_t == 0:
284+
d_t = 1
285+
if d_b == 0:
286+
d_b = 1
287+
288+
t = float(float(n_t/d_t) + float(n_b/d_b))
289+
log_string = 'UTILITY: '+webpage +' :TIME= '+str( n_t/float(d_t) )+':BW='+str( n_b/float(d_b))
290+
logger_1.info(log_string)
291+
logger_2.info(log_string)
292+
if t > 1:
293+
t = 1/t
294+
PREFETCHING_QUEUE.put((t, webpage))
295+
return PREFETCHING_QUEUE
296+
297+
298+

0 commit comments

Comments
 (0)