making http requests really parallel (1b799ea6) · Commits · Elphel / elphel-apps-autocampars

src/autocampars.py

+40 −98

Original line number	Original line	Diff line number	Diff line
	@@ -35,35 +35,26 @@ import os
	import urlparse		import urlparse
	import urllib		import urllib
	import time		import time
	import socket		#import shutil
	import shutil
	import sys		import sys
	import subprocess		import subprocess
	import select
	import urllib2		import urllib2
	import xml.etree.ElementTree as ET		import xml.etree.ElementTree as ET
			import threading
			import Queue

	import time		import time
	import random		import random
	PYDIR = "/usr/local/bin"		PYDIR = "/usr/local/bin"
	VERILOG_DIR = "/usr/local/verilog"		VERILOG_DIR = "/usr/local/verilog"
	WWW_SCRIPT="autocampars.py"		WWW_SCRIPT="autocampars.py"
	# shout(pydir+"/test_mcntrl.py @"+verilogdir+"/hargs-eyesis")		TIMEOUT = 20 # seconds
	"""
	TEST_URLS = ["http://xmission.com",
	"http://www3.elphel.com",
	"http://wiki.elphel.com"]
	"""
	def process_py393(args):		def process_py393(args):
	# print ('process_py393 (',args,')')
	# print (args['include'][0])
	cmd=PYDIR+'/test_mcntrl.py'		cmd=PYDIR+'/test_mcntrl.py'
	# print('0. calling ', cmd)
	try:		try:
	cmd +=' @'+VERILOG_DIR+'/'+args['include'][0]		cmd +=' @'+VERILOG_DIR+'/'+args['include'][0]
	except:		except:
	pass		pass
	# print('1. calling ', cmd)
	try:		try:
	if len(args['other']) > 0:		if len(args['other']) > 0:
	cmd +=' '+args['other'][0]		cmd +=' '+args['other'][0]
	@@ -76,21 +67,16 @@ def process_py393(args):
	fmt="""<?xml version="1.0"?>		fmt="""<?xml version="1.0"?>
	<result>%d</result>		<result>%d</result>
	"""		"""
	# print('fmt=',fmt)
	"""
	b=0;
	with open ('/dev/urandom','rb') as randf:
	for _ in range(4):
	b = (b << 8)+ord(randf.read(1))
	random.seed(b)
	time.sleep(random.random()*5.0)
	"""
	print (fmt%(rslt))		print (fmt%(rslt))
	# print ('<!--random.random()5.0 = ',random.random()5.0,' -->')
	print ('<!--process_py393 (',args,')-->')		print ('<!--process_py393 (',args,')-->')
	print('<!-- called ',cmd,'-->')		print('<!-- called ',cmd,'-->')

	def remote_parallel393(args, host, timeout=0): #fp in sec		def remote_parallel393(args, host, timeout=0): #fp in sec
			if not isinstance(host,(list,tuple)):
			if not isinstance(args,(list,tuple)):
			host=[host]
			else:
			host=[host]*len(args)
	if not isinstance(args,(list,tuple)):		if not isinstance(args,(list,tuple)):
	args=[args]*len(host)		args=[args]*len(host)
	argshosts=zip(args,host)		argshosts=zip(args,host)
	@@ -100,93 +86,49 @@ def remote_parallel393(args, host, timeout=0): #fp in sec
	for i in argshosts:		for i in argshosts:
	i[0]['cmd'] = 'py393'		i[0]['cmd'] = 'py393'
	urls.append("http://"+i[1]+"/"+WWW_SCRIPT+"?"+urllib.urlencode(i[0]))		urls.append("http://"+i[1]+"/"+WWW_SCRIPT+"?"+urllib.urlencode(i[0]))
	"""
	try:
	urls += TEST_URLS
	except:
	pass
	print ('urls=',urls)
	"""
	return remote_parallel_urls(urls=urls, timeout=timeout)		return remote_parallel_urls(urls=urls, timeout=timeout)

	def remote_parallel_urls(urls, timeout=0): #fp in sec		def remote_parallel_urls(urls, timeout=0): #imeout will restart for each next url
	rqs=[]		def read_url(index, queue, url):
	pending= [True]*len(urls)		queue.put((index, urllib2.urlopen(url).read()))
	rslts = [None]*len(urls)
	for url in urls:
	time.sleep(random.random()*1.0)
	rqs.append(urllib2.urlopen(url))
	while any(pending): #implement timeout
	if (timeout):
	# if True:
	ready_to_read, _, _ = select.select( #ready_to_write, in_error
	[rqs[i] for i,t in enumerate(pending) if t], # potential_readers,
	[], # potential_writers,
	[],
	timeout) # potential_errs,
	else:
	ready_to_read, _, _ = select.select( #ready_to_write, in_error
	[rqs[i] for i,t in enumerate(pending) if t], # potential_readers,
	[], # potential_writers,
	[]) # potential_errs,
	print('len(ready_to_read)= ',len(ready_to_read))
	if not ready_to_read: # timeout
	# break
	print('==== @ ',time.time())
	continue
	#find indices of the ready
	for fd in ready_to_read:
	fi=rqs.index(fd)
	print('--Got response from ',urls[fi],' (',fi,') @ ',time.time())
	rslts[fi] = r=fd.read()
	pending[fi]=False
	print('results=',rslts)
	return rslts
	#if None in rslts - likely timeout happened

	def remote_py393(args, host):		queue = Queue.Queue()
	print ("remote_py393(",args,',',host,')')		for index, url in enumerate(urls):
	if isinstance(host,(list,tuple)):		thread = threading.Thread(target=read_url, args = (index, queue, url))
	rslts = remote_parallel393(args,host)		thread.daemon = True
	else:		thread.start()
	args['cmd'] = 'py393'		rslts=[None]*len(urls)
	uea = urllib.urlencode(args)		if not timeout:
	url="http://"+host+"/"+WWW_SCRIPT+"?"+uea		timeout = None
	print ("opening url:",url)		for _ in urls:
	rslts = [urllib2.urlopen(url).read()]
	print('Received:',rslts)
	for i, r in enumerate(rslts):
	try:		try:
	rslts[i] = int(ET.fromstring(r).text)		rslt = queue.get(block=True, timeout=timeout)
			rslts[rslt[0]] = rslt[1]
	except:		except:
	rslts[i] = -1		break
	return rslts		return rslts
			#if None in rslts - likely timeout happened


	def remote_wget(url, host, timeout=0) : #split_list_arg(sys.argv[1]))		def remote_wget(url, host, timeout=0) : #split_list_arg(sys.argv[1]))
	print ("remote_wget(",args,',',host,')')		print ("remote_wget(",args,',',host,')')
	if not isinstance(host,(list,tuple)):		if not isinstance(host,(list,tuple)):
	host=[host]		host=[host]
	if (len(host)>1) or (len(url)>1):
	if len(host) > len(url):		if len(host) > len(url):
	url+= [url[-1]]*(len(host) - len(url))		url+= [url[-1]]*(len(host) - len(url))
	elif len(url) > len(url):		elif len(url) > len(url):
	host += [host[-1]]*(len(url) - len(host))		host += [host[-1]]*(len(url) - len(host))

	argshosts=zip(args,host)		argshosts=zip(args,host)
	print ('remote_parallel393(args=',args,' host=',host,')')		print ('remote_wget(args=',args,' host=',host,')')
	print ('argshosts=',argshosts)		print ('argshosts=',argshosts)
	urls=[]		urls=[]
	for i in argshosts:		for i in argshosts:
	urls.append("http://"+i[1]+"/"+i[0])		urls.append("http://"+i[1]+"/"+i[0])
	rslts=remote_parallel_urls(urls=urls, timeout=timeout)		rslts=remote_parallel_urls(urls=urls, timeout=timeout)
	else:
	print ("opening url:",url)
	rslts = [urllib2.urlopen(url).read()]
	print('Received:'+rslts)
	# parse results here		# parse results here
	return rslts		return rslts


	def process_http():		def process_http():
	try:		try:
	qs=urlparse.parse_qs(os.environ['QUERY_STRING'])		qs=urlparse.parse_qs(os.environ['QUERY_STRING'])
	@@ -222,7 +164,7 @@ def process_cmdline():
	except:		except:
	pass		pass
	print ('args=',args)		print ('args=',args)
	remote_py393(args, split_list_arg(sys.argv[1]))		remote_parallel393 (args=args, host = split_list_arg(sys.argv[1]), timeout=TIMEOUT)
	elif sys.argv[2] == "wget":		elif sys.argv[2] == "wget":
	urls = sys.argv[3:]		urls = sys.argv[3:]
	remote_wget(urls, split_list_arg(sys.argv[1]))		remote_wget(urls, split_list_arg(sys.argv[1]))

Admin message