python 2 to 3 migration

Makefile

@@ -24,7 +24,7 @@ CXX_WARN ?= \
 	-Wall -Werror -Wpedantic -Wno-vla -Wno-sign-compare \
 	-Wno-unused-but-set-variable -Wno-unused-variable \
 	-Wno-unused-function
-PYTHON ?= python2
+PYTHON ?= python3
 MPICXX ?= mpicxx
 MPICH_CXX?=${CXX}
 CXX_FLAGS += \

scripts/common.py

-#!/usr/bin/python
+#!/usr/bin/python3
 
 class FatalError(Exception):
 	def __init__(self, message, *args, **kwargs):

scripts/explore_log_cfs.py

@@ -58,7 +58,7 @@ def read_cfs(filename):
   for i,s in read_file_lines(filename):
     rank,time,event,description = get_rank_time_event_description(s)
     if event == None:
-      print(i,s)
+      print((i,s))
     if event in events_cf:
       cfname = get_cf_name(s)[0]
       key = (cfname,rank)
@@ -95,7 +95,7 @@ def output_cfs(filename,cfs):
 #------------------------------------------------------------------
 
 if len(sys.argv) == 1:
-  print("Usage:",sys.argv[0],"<filename>")
+  print(("Usage:",sys.argv[0],"<filename>"))
   sys.exit()
 
 logfilename = sys.argv[1]

scripts/explore_log_dfs.py

@@ -58,7 +58,7 @@ def read_dfs(filename):
   for i,s in read_file_lines(filename):
     rank,time,event,description = get_rank_time_event_description(s)
     if event == None:
-      print(i,s)
+      print((i,s))
     if event in events_df:
       dfname = get_df_name(s)[0]
       key = (dfname,rank)
@@ -93,7 +93,7 @@ def split_dfs_resp_unresp(dfs):
       newdf = [ df[IDFNAME], df[IRANK], [], len(reqs) ]
       n = 0
       for event in df[IEVENTS]:
-        if n in reqs.values():
+        if n in list(reqs.values()):
           newevent = ( event[0],event[1],event[2],1 )
         else:
           newevent = ( event[0],event[1],event[2],0 )
@@ -124,7 +124,7 @@ def output_dfs_unresp(filename,dfs):
 #------------------------------------------------------------------
 
 if len(sys.argv) == 1:
-  print("Usage:",sys.argv[0],"<filename>")
+  print(("Usage:",sys.argv[0],"<filename>"))
   sys.exit()
 
 logfilename = sys.argv[1]

scripts/fcmp

-#!/usr/bin/python
+#!/usr/bin/python3
 
 # COMPILATION OPTIONS
 
@@ -342,12 +342,12 @@ class RegFile:
 
 	def name_info(self, name):
 		res='%s:' % name
-		for kind, spec in self._names[name].iteritems():
+		for kind, spec in self._names[name].items():
 			if spec is None:
 				res+=' %s' % kind
 			else:
 				res+=' %s{' % kind + ' '.join(['%s=%s' % (k, repr(v)) \
-					for k, v in spec.iteritems()]) + '}'
+					for k, v in spec.items()]) + '}'
 		return res
 
 	def alloc(self, comment):
@@ -410,7 +410,7 @@ class RegFile:
 			elif type(val)==list:
 				val_class='symbolic'
 				val={'type': 'id', 'ref': val}
-			elif type(val) in [str, unicode]:
+			elif type(val) in [str, str]:
 				val_class='generated'
 			else:
 				raise Exception(val, val.__class__)
@@ -448,7 +448,7 @@ class RegFile:
 		if id.Name in self.Locators:
 			l=self.Locators[id.Name]
 			if len(id.Indices)!=len(l['vars']):
-				print id, l['vars']
+				print(id, l['vars'])
 				raise Exception('idx length missmatch')
 
 			params={}
@@ -462,7 +462,7 @@ class RegFile:
 			
 			res=json.loads(json.dumps(l['expr']))
 
-			for k, v in params.iteritems():
+			for k, v in params.items():
 				regs.set_param(k, v)
 			return res, params
 		elif self.Parent is not None:
@@ -480,7 +480,7 @@ class RegFile:
 		if id.Name in self.Locators:
 			l=self.Locators[id.Name]
 			if len(id.Indices)!=len(l['vars']):
-				print id, l['vars']
+				print(id, l['vars'])
 				sys.stderr.write('%s %s\n' % (repr(id), repr(l['vars'])))
 				raise Exception('idx length missmatch')
 
@@ -492,7 +492,7 @@ class RegFile:
 				params[key]=regs.set_param(key, expr.gen(regs))
 			
 			res=LocatorCyclicExpr(create_expr(l['expr'], regs)).gen(regs)
-			for k, v in params.iteritems():
+			for k, v in params.items():
 				regs.set_param(k, v)
 			return res
 		elif self.Parent is not None:
@@ -523,7 +523,7 @@ class RegFile:
 				res=self.get_name_reg(name)
 				return res
 			else:
-				print self
+				print(self)
 				raise Exception('Name not recognized', name)
 		else:
 			# According to name lookup priority:
@@ -575,13 +575,13 @@ class RegFile:
 				else:
 					return spec['sub_param']['reg']
 			else:
-				print self._names[name]
-				print self.name_info(name)
+				print(self._names[name])
+				print(self.name_info(name))
 				R(name)
 		R()
 
 	def imported(self, parent_reg):
-		for reg, spec in self._regs.iteritems():
+		for reg, spec in self._regs.items():
 			if spec['type']=='import' and spec['parent_reg']==parent_reg:
 				return reg
 		# not imported, import
@@ -593,7 +593,7 @@ class RegFile:
 		return self.imported(parent_reg)
 
 	def imported_lc(self, ctr_name):
-		for reg, spec in self._regs.iteritems():
+		for reg, spec in self._regs.items():
 			if spec['type']=='import_lc':
 				return reg
 		# not imported, import
@@ -605,7 +605,7 @@ class RegFile:
 		return self.imported_lc(ctr_name)
 
 	def arg_reg(self, arg_num, comment=None):
-		for reg, spec in self._regs.iteritems():
+		for reg, spec in self._regs.items():
 			if spec['type']=='arg':
 				return reg
 		# no such arg, alloc
@@ -633,7 +633,7 @@ class RegFile:
 		elif pv[0]=='parent_while_counter':
 			return ('reg', self.arg_reg(0))
 		else:
-			print pv
+			print(pv)
 			raise Exception(pv)
 
 	def get_value(self, id):
@@ -641,8 +641,8 @@ class RegFile:
 			if self.Parent is not None:
 				return self.get_parent_value(id)
 			else:
-				print "-------------------"
-				print self
+				print("-------------------")
+				print(self)
 				raise NameNotRecognized(id.Name)
 
 		spec=self._names[id.Name]
@@ -714,14 +714,14 @@ class RegFile:
 				return ('name_reg', spec['cf']['reg'], id)
 				
 		else:
-			print "NAME >> ", id
-			print self
+			print("NAME >> ", id)
+			print(self)
 			raise Exception('add according to priority', spec)
 
 		R(self.name_info(id.Name))
 		vt=self._get_value_type(id)
-		print vt
-		print self
+		print(vt)
+		print(self)
 		R(repr(id))
 
 	def reg_info(self, reg):
@@ -1524,7 +1524,7 @@ class Sub(Scope):
 					and id.Name==rule['id'][0]:
 				if id.Indices or rule['id'][1:]:
 					if len(id.Indices)!=len(rule['id'][1:]):
-						print id
+						print(id)
 						R(rule['id'])
 					params=kwargs.get('subst', {})
 					for i in range(len(rule['id'][1:])):
@@ -1653,13 +1653,13 @@ class Fa:
 		return asm
 
 def show_help():
-	print '''\
+	print('''\
 Usage:
-	python %s input.ja output.fasm [--cpp-codes cpp_codes] [--only-requests]
+	python3 %s input.ja output.fasm [--cpp-codes cpp_codes] [--only-requests]
 	(will generate cpp_codes.h and cpp_codes.cpp if --cpp-codes is set)
 	With --only-requests nothing will happen except writing a modified
 	json with additional rules (posts/requests unlimited)
-''' % sys.argv[0]
+''' % sys.argv[0])
 
 if '--help' in sys.argv[1:]:
 	show_help()

scripts/fcmp2

-#!/usr/bin/python
+#!/usr/bin/python3
 
 DEFAULT_LOCATOR_0=True # Set locator 0 if unset
 ADD_DEBUG_INFO=False
@@ -98,10 +98,10 @@ def create_scope(j, parent):
 	return scope
 
 def ps(scope):
-	for k, v in scope.iteritems():
+	for k, v in scope.items():
 		if k in ['j', 'parent', 'children']:
 			continue
-		print '%s=%s' % (k, repr(v))
+		print('%s=%s' % (k, repr(v)))
 
 class UnresolvedName(Exception):
 	def __init__(self, *args, **kwargs):
@@ -1102,7 +1102,7 @@ def parse_import(ja):
 
 if __name__=='__main__':
 	if '--help' in sys.argv[1:]:
-		print HELP_MESSAGE
+		print(HELP_MESSAGE)
 		sys.exit(0)
 	if len(sys.argv[1:])<3:
 		sys.stderr.write('%s: invalid arguments count, see --help\n' \
@@ -1131,7 +1131,7 @@ if __name__=='__main__':
 		mid+=BLOCKH % ('MAIN', main_id)
 
 
-		for name, sub in gja.iteritems():
+		for name, sub in gja.items():
 			assert 'name' not in sub
 			sub['name']=name
 

scripts/luna

-#!/usr/bin/python
+#!/usr/bin/python3
 
 # TODO add ability to use some other temporary directory for building
 # tests, not .luna subdirectory, and allow setup via environment var.
@@ -172,7 +172,7 @@ def parse_env(conf):
 			os.pardir))
 	
 	if 'PYTHON' not in conf:
-		conf['PYTHON']=os.environ.get('PYTHON', 'python')
+		conf['PYTHON']=os.environ.get('PYTHON', 'python3')
 
 	if 'CXX_FLAGS' not in conf and 'CXX_FLAGS' in os.environ:
 		conf['CXX_FLAGS']=os.environ['CXX_FLAGS']
@@ -231,7 +231,7 @@ def generate_cpp_blocks(ja_in_path, headers_in_path, ja_ti_path, b_in_path,
 		else:
 			raise NotImplementedError(header[0], header)
 
-	for name, sub in ja.iteritems():
+	for name, sub in ja.items():
 		if sub['type']=='foreign_cpp':
 			block_id=sub['block_id']
 
@@ -274,7 +274,7 @@ def info(msg):
 		if msg is None:
 			return
 		else:
-			print msg
+			print(msg)
 	else:
 		if msg is None:
 			sys.stdout.write('\033[2K\r')
@@ -296,7 +296,7 @@ def error_block_message(block_id, blocks_info, pp_ti):
 		mark(line, pos))
 
 def so_build_message(eln, epos, emsg):
-	emsg=emsg.decode('utf-8')
+	#emsg=emsg.decode('utf-8')
 	ln=1
 	cur=1
 	for i in range(len(conf['fb.ti']['text'])):
@@ -305,7 +305,7 @@ def so_build_message(eln, epos, emsg):
 		if ln==eln and cur==epos:
 			C=conf['fb.ti']['text'][i]
 			mrk=mark(conf['fb.cpp'][ln-1], cur-1)
-			return u'compile: at %s:%d:%d: %s\n%s' % (
+			return 'compile: at %s:%d:%d: %s\n%s' % (
 				conf['fb.ti']['paths'][str(C[0])], C[1], C[2],
 				emsg, mrk)
 		if conf['fb.ti']['text'][i][3]=='\n':
@@ -338,17 +338,17 @@ def translate_so_build_err(err):
 			elif m2 is not None:
 				bid=int(m2.groups()[0])
 				C=conf['blocks.ti']['text'][bid][0]
-				res.append(u'in foreign block at %s:%d:%d:\n%s' % (
+				res.append('in foreign block at %s:%d:%d:\n%s' % (
 					conf['blocks.ti']['paths'][str(C[0])],C[1], C[2],
 					mark(get_src(conf['blocks.ti']['paths'][str(C[0])])[C[1]], C[2])))
 			else:
-				res.append(ln.decode('utf-8'))
+				res.append(ln)#.decode('utf-8'))
 		else:
-			res.append(ln.decode('utf-8'))
+			res.append(ln)#.decode('utf-8'))
 
 		cur+=1
 	
-	return u'\n'.join(res)
+	return '\n'.join(res)
 
 
 def translate_fcmp2_err(err):
@@ -374,7 +374,7 @@ def translate_fcmp2_err(err):
 def main():
 	global conf, ti
 	if '--help' in sys.argv[1:]:
-		print HELP_MESSAGE
+		print(HELP_MESSAGE)
 		sys.exit(0)
 
 	conf=parse_args(sys.argv[1:])
@@ -390,7 +390,7 @@ def main():
 		conf['BUILD_DIR']=os.path.join(conf['LUNA_HOME'], 'build', 'programs', *dirs)
 
 	if 'PYTHON' not in conf:
-		conf['PYTHON']='python'
+		conf['PYTHON']='python3'
 
 	conf['CXX_FLAGS']=conf.get('CXX_FLAGS', '') \
 		+ ' -I %s' % os.path.join(conf['LUNA_HOME'], 'include') \
@@ -443,13 +443,15 @@ def main():
 			'blocks.ti')]
 
 		if VERBOSE_FLAG:
-			print ' '.join(cmd)
+			print(' '.join(cmd))
 		p=subprocess.Popen(cmd, stderr=subprocess.PIPE,
 			stdin=subprocess.PIPE, stdout=subprocess.PIPE)
 		out, err=p.communicate()
+		out=out.decode('utf-8')
+		err=err.decode('utf-8')
 
 		if VERBOSE_FLAG:
-			print out
+			print(out)
 
 		if p.returncode!=0:
 			raise FatalError('preprocessing failed (see error messages ' \
@@ -468,9 +470,11 @@ def main():
 		cmd+=['-o', os.path.join(conf['BUILD_DIR'], 'program.ja')]
 		cmd+=['-h', os.path.join(conf['BUILD_DIR'], 'headers.ja')]
 		if VERBOSE_FLAG:
-			print ' '.join(cmd)
+			print(' '.join(cmd))
 		p=subprocess.Popen(cmd, stderr=subprocess.PIPE, stdout=subprocess.PIPE)
 		out, err=p.communicate()
+		out=out.decode('utf-8')
+		err=err.decode('utf-8')
 
 		if p.returncode!=0:
 			for ln in err.split('\n'):
@@ -483,7 +487,7 @@ def main():
 				+ 'see error messages below:\n%s' % err)
 
 		if VERBOSE_FLAG:
-			print out
+			print(out)
 
 		# ================
 		# let substitution
@@ -495,17 +499,19 @@ def main():
 			'substitution_let.py')]
 		cmd+=[os.path.join(conf['BUILD_DIR'], 'program.ja')]
 		if VERBOSE_FLAG:
-			print ' '.join(cmd)
+			print(' '.join(cmd))
 		p=subprocess.Popen(cmd, stderr=subprocess.PIPE,
 			stdout=subprocess.PIPE)
 		out, err=p.communicate()
+		out=out.decode('utf-8')
+		err=err.decode('utf-8')
 
 		if p.returncode!=0:
 			raise FatalError('compilation failed: ' \
 				+ 'see error messages below:\n%s' % err)
 
 		if VERBOSE_FLAG:
-			print out
+			print(out)
 
 		# ===================
 		# generate cpp blocks
@@ -546,16 +552,18 @@ def main():
 			os.path.join(conf['BUILD_DIR'], 'program_recom.ja'),
 			'--only-requests']
 		if VERBOSE_FLAG:
-			print ' '.join(cmd)
+			print(' '.join(cmd))
 		p=subprocess.Popen(cmd, stderr=subprocess.PIPE, stdout=subprocess.PIPE)
 		out, err=p.communicate()
+		out=out.decode('utf-8')
+		err=err.decode('utf-8')
 
 		if p.returncode!=0:
 			raise FatalError('recom-generation failed (see below):\n%s' \
 				% err)
 
 		if VERBOSE_FLAG:
-			print out
+			print(out)
 
 		# ================
 		# Generate ja->cpp
@@ -574,16 +582,18 @@ def main():
 			cmd.append('--add-debug-info')
 
 		if VERBOSE_FLAG:
-			print ' '.join(cmd)
+			print(' '.join(cmd))
 		p=subprocess.Popen(cmd, stderr=subprocess.PIPE, stdout=subprocess.PIPE)
 		out, err=p.communicate()
+		out=out.decode('utf-8')
+		err=err.decode('utf-8')
 
 		if p.returncode!=0:
 			raise FatalError('cpp-generation failed (see below):\n%s' \
 				% translate_fcmp2_err(err))
 		
 		if VERBOSE_FLAG:
-			print out
+			print(out)
 
 		cpp_blocks_info=json.loads(open(
 			os.path.join(conf['BUILD_DIR'], 'cpp_blocks_info.json')).read())
@@ -605,7 +615,7 @@ def main():
 		]
 
 		if VERBOSE_FLAG:
-			print ' '.join(cmd)
+			print(' '.join(cmd))
 		p=subprocess.Popen(cmd, stderr=subprocess.PIPE, stdout=subprocess.PIPE,
 			stdin=subprocess.PIPE)
 
@@ -613,7 +623,9 @@ def main():
 			for f in os.listdir(conf['PROGRAM_DIR']) if f.endswith('.cpp')]
 		cpp_list+=[os.path.join(conf['BUILD_DIR'], f) \
 			for f in os.listdir(conf['BUILD_DIR']) if f.endswith('.cpp')]
-		out, err=p.communicate(input='\n'.join(cpp_list))
+		out, err=p.communicate(input=('\n'.join(cpp_list)).encode('utf-8'))
+		out=out.decode('utf-8')
+		err=err.decode('utf-8')
 
 		if p.returncode!=0:
 			raise FatalError('makefile-generation failed (see below):\n%s' \
@@ -633,11 +645,13 @@ def main():
 			os.path.join(conf['BUILD_DIR'], 'Makefile.libucodes')]
 
 		if VERBOSE_FLAG:
-			print ' '.join(cmd)
+			print(' '.join(cmd))
 			p=subprocess.Popen(cmd, stderr=subprocess.PIPE)
 		else:
 			p=subprocess.Popen(cmd, stderr=subprocess.PIPE, stdout=subprocess.PIPE)
 		out, err=p.communicate()
+		out=out.decode('utf-8')
+		err=err.decode('utf-8')
 
 		if p.returncode!=0:
 			err=translate_so_build_err(err)
@@ -645,7 +659,7 @@ def main():
 				% (p.returncode, err))
 
 		if VERBOSE_FLAG:
-			print out
+			print(out)
 		
 		info('running program')
 		info(None)
@@ -664,7 +678,7 @@ def main():
 		
 
 		if VERBOSE_FLAG:
-			print ' '.join(cmd)
+			print(' '.join(cmd))
 		t0=datetime.datetime.now()
 		p=subprocess.Popen(cmd, stderr=subprocess.PIPE, env=env)
 
@@ -673,6 +687,10 @@ def main():
 		signal.signal(signal.SIGINT, signal.SIG_IGN)
 
 		out, err=p.communicate()
+		if out is not None:
+			out=out.decode('utf-8')
+		if err is not None:
+			err=err.decode('utf-8')
 
 		signal.signal(signal.SIGINT, old_handler)
 
@@ -693,11 +711,11 @@ def main():
 						m.groups()[0]))
 				if not recognized_flag:
 					msgs.append('err> %s' % ln)
-			raise FatalError(u'run-time error: errcode=%d' % p.returncode \
-				+ u''.join([u'\n%s' % msg for msg in msgs]))
+			raise FatalError('run-time error: errcode=%d' % p.returncode \
+				+ ''.join(['\n%s' % msg for msg in msgs]))
 
 		if conf['TIME']:
-			print "TIME: %lf sec." % (t-t0).total_seconds()
+			print("TIME: %lf sec." % (t-t0).total_seconds())
 		
 	finally:
 		if conf['CLEANUP']:
@@ -705,7 +723,7 @@ def main():
 			try:
 				cmd=['rm', '-r', conf['BUILD_DIR']]
 				if VERBOSE_FLAG:
-					print ' '.join(cmd)
+					print(' '.join(cmd))
 				subprocess.check_call(cmd)
 			except subprocess.CalledProcessError as x:
 				raise FatalError('cleanup failed (errcode=%d)' % x.returncode)
@@ -716,7 +734,7 @@ if __name__=='__main__':
 	try:
 		main()
 	except FatalError as x:
-		sys.stderr.write(('%s: fatal error: %s\n' % (_argv0, x.message)).encode('utf-8'))
+		sys.stderr.write(('%s: fatal error: %s\n' % (_argv0, x.message)))
 		sys.exit(1)
 	except CompileError as x:
 		sys.stderr.write('%s: compile error: %s at %s:%d\n' % (

scripts/mkgen

-#!/usr/bin/python
+#!/usr/bin/python3
 
 import subprocess, os, sys
 
@@ -48,7 +48,7 @@ def test_parse_args():
 		=={'--out': '4', '--compile-flags': '234 234 234'}
 
 def check_args(args):
-	for k, v in KEYS.iteritems():
+	for k, v in KEYS.items():
 		if k in args:
 			continue
 		if 'default' not in v:
@@ -64,13 +64,13 @@ def is_ext_valid(path, xlist):
 test_parse_args()
 
 if '--help' in sys.argv[1:]:
-	print "A 'one-dir' makefile generator"
-	print "Supported keys:"
-	for k, v in KEYS.iteritems():
+	print("A 'one-dir' makefile generator")
+	print("Supported keys:")
+	for k, v in KEYS.items():
 		if 'default' in v:
-			print '\t%s=<%s> (default: "%s")' % (k, v['help'], v['default'])
+			print('\t%s=<%s> (default: "%s")' % (k, v['help'], v['default']))
 		else:
-			print '\t%s=<%s>' % (k, v['help'])
+			print('\t%s=<%s>' % (k, v['help']))
 	sys.exit(0)
 
 args=parse_args(sys.argv[1:])
@@ -78,7 +78,7 @@ check_args(args)
 
 objs=[]
 
-print "default:", args['--out']
+print("default:", args['--out'])
 
 def get_sources(args):
 	if args['--src-stdin']:
@@ -97,7 +97,7 @@ for f in get_sources(args):
 		cmd+=(args['--compile-flags'].split(' '))
 	try:
 		cmd=[x for x in cmd if x]
-		out=subprocess.check_output(cmd)
+		out=subprocess.check_output(cmd).decode('utf-8')
 	except subprocess.CalledProcessError as x:
 		sys.exit(1)
 	except OSError as x:
@@ -106,17 +106,17 @@ for f in get_sources(args):
 	obj, out=out.split(':')
 	out=out.replace(' \\\n', ' ').strip()
 	deps=out.split(' ')
-	print os.path.join(args['--obj'], obj),':', ' '.join(deps)
-	print '\t'+args['--compiler'], args['--compile-flags'], \
+	print(os.path.join(args['--obj'], obj),':', ' '.join(deps))
+	print('\t'+args['--compiler'], args['--compile-flags'], \
 		'-c', os.path.join(args['--src'], f), \
-		'-o', os.path.join(args['--obj'], obj)
+		'-o', os.path.join(args['--obj'], obj))
 	objs.append(os.path.join(args['--obj'], obj))
 
-print
-print args['--out'], ':', ' '.join(objs)
-print '\t'+args['--compiler'], '-o', \
-	args['--out'], ' '.join(objs), args['--link-flags']
+print()
+print(args['--out'], ':', ' '.join(objs))
+print('\t'+args['--compiler'], '-o', \
+	args['--out'], ' '.join(objs), args['--link-flags'])
 
-print
-print 'clean:'
-print '\trm -f', args['--out'], ' '.join(objs)
+print()
+print('clean:')
+print('\trm -f', args['--out'], ' '.join(objs))