luna 19.2 KB
Newer Older
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
1
#!/usr/bin/python3
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
2
3
4
5

# TODO add ability to use some other temporary directory for building
# tests, not .luna subdirectory, and allow setup via environment var.

Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
6
import sys, os, subprocess, json, re, datetime, signal
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
7
from common import *
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
8
9
10
11
12
13
14
15

_argv0=os.path.split(sys.argv[0])[1]
HELP_MESSAGE=sys.argv[0]+'''

NAME
	''' + _argv0 + ''' - execute LuNA program

SYNOPSIS
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
16
	''' + _argv0 + ''' [OPTION]... FILE [ARGUMENT]...
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
17
18

DESCRIPTION
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
19
	Executes LuNA program, described in FILE.
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
20

Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
21
22
23
24
	FILE is a path of LuNA-program (*.fa) to execute. In the LuNA-
	program subroutine called 'main' is executed. If the subroutine
	has arguments then corresponding number of arguments must be
	passed.
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
25

Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
26
27
OPTIONS
	--help
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
28
29
30
31
32
33
34
35
36
37
38
	    Show this help and exit. Other options are ignored.
	
	--build-dir=<path>
	    Specify path for intermediate files' dir. Warning: may
	    be deleted with all the contents. Default is './.luna'.
	
	--luna-home=<path>
	    Specify luna system installation dir (overrides LUNA_HOME env var).
	
	--disable-warning=<NUM>
	    Do not show warnings with given number.
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
39
	
40
41
42
	-q
	--quiet
		Do not show progress info.
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
43
44
45
46

	-g
	    Include debug information (may reduce performance)
	
a-chmil's avatar
a-chmil committed
47
48
49
	-b
		Enable dynamic balance.
		
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
50
51
52
53
54
55
56
57
	-O0
	-O
	    Specify optimization level. -O0 optimizes compilation time at
	    the cost of performance reduction. -O optimizes performance
	    (may increase compilation time).
	
	--no-cleanup
	    Do not delete intermediate files after program termination.
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
58
59
60

	--verbose
	    Display most intermediate info
61
	
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
62
63
	-t
	    Show program execution time (compilation time is not included).
64
65
66
67

	--compile-only
	    Do not run program after compilation. Also sets no-cleanup 
	    flag. Output is a command for program execution.
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
68
	
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
69
70
71
72
73
ENVIRONMENT
	Environment variables affect LuNA settings, but the OPTIONs take
	precedence over them (if appropirate).

	LUNA_HOME
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
74
75
		Path to LuNA home directory. If unset, LUNA_HOME is set to
		'<script_dir>/..'.
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
	
	PYTHON
	    Python interpreter to use.

	CXX_FLAGS
	    Additional flags used to compile program sources (*.cpp).

	CXX
	    C++ compiler to use to compile program sources (*.cpp).

	LDFLAGS
	    C++ linker flags to link program sources.
	
	LUNA_NO_CLEANUP
		If this variable is present, then temporary build directory
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
91
	    will not be removed in the end (same as --no-cleanup option).
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
92
93
94
95
96

REPORTING BUGS
	Report bugs to perepelkin@ssd.sscc.ru.
'''

Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
97
98
99
100
101
102
103
class CompileError(Exception):
	def __init__(self, message, ti, pos):
		self.message=message
		self.path=ti['paths'][str(ti["text"][pos][0])]
		self.line_num=ti["text"][pos][1]
		self.line_pos=ti["text"][pos][2]
		self.line=open(self.path).read().split('\n')[self.line_num]
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
104

Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
105
DISABLED_WARNINGS=set()
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
106

Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
107
108
def warn(num, msg):
	if num in DISABLED_WARNINGS:
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
109
		return
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
110
	sys.stderr.write('%s: WARNING#%d: %s\n' % (_argv0, num, msg))
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
111
112

def parse_args(args):
113
114
115
	'''
	Parse and verify command-line arguments (see --help for info)
	'''
Anton Kudinov's avatar
Anton Kudinov committed
116
	global QUIET_FLAG, VERBOSE_FLAG, COMPILE_ONLY_FLAG
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
117
118
	conf={}
	cur=0
119
120

	# Default config parameters
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
121
122
	QUIET_FLAG=False
	conf['DEBUG']=False
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
123
	conf['CLEANUP']=True
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
124
	conf['TIME']=False
125
	conf['BALANCE']=False
126
	VERBOSE_FLAG=False
Anton Kudinov's avatar
Anton Kudinov committed
127
128
	COMPILE_ONLY_FLAG=False

129

Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
130
131
132
133
134
135
136
137
138
139
140
141
142
143
	while cur<len(args):
		arg=args[cur]
		
		if arg.startswith('--build-dir='):
			conf['BUILD_DIR']=arg.split('=', 1)[1]
		elif arg.startswith('--luna-home='):
			conf['LUNA_HOME']=arg.split('=', 1)[1]
		elif arg.startswith('--disable-warning='):
			try:
				num=int(arg.split('=', 1)[1])
				DISABLED_WARNINGS.add(num)
			except ValueError:
				raise FatalError("'--disable-warning' requires "\
					+ "numerical argument, got: %s" % arg.split('=', 1)[1])
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
144
145
		elif arg in ['-q', '--quiet']:
			QUIET_FLAG=True
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
146
		elif arg=='--verbose':
147
			VERBOSE_FLAG=True
Anton Kudinov's avatar
Anton Kudinov committed
148
149
150
		elif arg=='--compile-only':
			COMPILE_ONLY_FLAG=True
			conf['CLEANUP']=False
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
151
152
153
154
155
156
157
		elif arg in ['-O', '-O0']:
			if 'OPTIMIZATION' in conf and conf['OPTIMIZATION']!=arg:
				raise FatalError("conflicting optimization options: " \
					+ "'%s' and '%s'" % (conf['OPTIMIZATION'], arg))
			conf['OPTIMIZATION']=arg
		elif arg=='-g':
			conf['DEBUG']=True
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
158
159
		elif arg=='--no-cleanup':
			conf['CLEANUP']=False
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
160
161
		elif arg=='-t':
			conf['TIME']=True
162
163
		elif arg=='-b':
			conf['BALANCE']=True
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
164
		else:
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
165
166
167
168
169
170
			if arg[cur].startswith('-'):
				warn(1, "suspicious program name: '%s' (mistyped a key?)" \
					% arg)
			conf['PROGRAM']=args[cur]
			conf['PROGRAM_DIR']=os.path.dirname(os.path.abspath(
				conf['PROGRAM']))
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
171
172
			if 'OPTIMIZATION' not in conf:
				conf['OPTIMIZATION']='-O'
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
173
174
175
176
			conf['ARGV']=args[cur+1:]
			return conf

		cur+=1
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
177

Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
178
179
180
	raise FatalError('no program specified')

def parse_env(conf):
181
182
183
	'''
	Check environment variables for unset config parameters
	'''
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
184
185
186
187
188
189
	if 'LUNA_HOME' not in conf:
		conf['LUNA_HOME']=os.environ.get('LUNA_HOME',
			os.path.join(os.path.dirname(os.path.realpath(__file__)),
			os.pardir))
	
	if 'PYTHON' not in conf:
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
190
		conf['PYTHON']=os.environ.get('PYTHON', 'python3')
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
191
192
193
194
195
196
197
198
199
200

	if 'CXX_FLAGS' not in conf and 'CXX_FLAGS' in os.environ:
		conf['CXX_FLAGS']=os.environ['CXX_FLAGS']

	if 'CXX' not in conf and 'CXX' in os.environ:
		conf['CXX']=os.environ['CXX']

	if 'LDFLAGS' not in conf and 'LDFLAGS' in os.environ:
		conf['LDFLAGS']=os.environ['LDFLAGS']
	
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
201
202
	if 'LUNA_NO_CLEANUP' in os.environ:
		conf['CLEANUP']=False
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
203
204
205
	
	if 'CXX' not in conf:
		conf['CXX']='g++'
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
206

207
208
209
210
211
212
213
C_TYPES={
	'int': 'int',
	'real': 'double',
	'string': 'const char *',
	'value': 'const InputDF &',
	'name': 'OutputDF &'
}
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
214

215
216
217
218
219
220
221
222
223
def get_path_id(path, id2path, path2id, SRC):
	if path in path2id:
		return path2id[path]
	else:
		id=len(path2id)
		path2id[path]=id
		id2path[id]=path
		return id

224
def generate_cpp_blocks(ja_in_path, headers_in_path, ja_ti_path, b_in_path,
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
225
226
		ja_out_path, b_out_path, b_ti_path):
	ja=json.loads(open(ja_in_path).read())
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
227
228
229
230
	try:
		hja=json.loads(open(headers_in_path).read())
	except IOError:
		hja=[]
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
231
232
233
234
235
236
237
238
239
	blocks=json.loads(open(b_in_path).read())

	path2id={}
	id2path={}
	SRC={}
	Cpp=[]

	for C in to_String(SRC, path2id, id2path, '#include "ucenv.h"\n\n'):
		Cpp.append(C)
240
241
242
243
244
245
246
247
	
	for header in hja:
		if header[0]=='head':
			block_id=header[1]
			for C in blocks['text'][block_id]:
				Cpp.append(C)
		else:
			raise NotImplementedError(header[0], header)
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
248

Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
249
	for name, sub in ja.items():
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
250
251
252
253
254
255
		if sub['type']=='foreign_cpp':
			block_id=sub['block_id']

			# Generate signature
			sig='extern "C" void __foreign_block_%d(' % block_id

256
257
			sig+=', '.join(['%s %s' % (C_TYPES[arg['type']], arg['id']) \
				for arg in sub['args']])
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
258
259
260
261
262
263
264
265
266

			sig+=')\n'

			for C in to_String(SRC, path2id, id2path, sig):
				Cpp.append(C)

			# Append block

			for C in blocks['text'][block_id]:
267
268
269
				path=blocks['paths'][str(C[0])]
				C[0]=get_path_id(path, id2path, path2id, SRC)

Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
270
271
272
				Cpp.append(C)

			sub['type']='extern'
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
273
			sub['foreign_type']='C++'
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
274
275
			sub['code']='__foreign_block_%d' % block_id
			del sub['block_id']
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
276
	
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
277
278
279
280
281
282
283
284
	open(b_out_path, 'w').write(to_string(Cpp))
	open(b_ti_path, 'w').write(json.dumps({
		"paths": id2path,
		"text": Cpp
	}))
	open(ja_out_path, 'w').write(json.dumps(ja))
				

285
def info(msg):
Anton Kudinov's avatar
Anton Kudinov committed
286
	if QUIET_FLAG or COMPILE_ONLY_FLAG:
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
287
		return
288
	if VERBOSE_FLAG:
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
289
290
291
		if msg is None:
			return
		else:
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
292
			print(msg)
293
	else:
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
294
295
296
297
298
299
		if msg is None:
			sys.stdout.write('\033[2K\r')
		else:
			assert '\n' not in msg
			sys.stdout.write('\033[2K\rluna: %s' % msg)
		sys.stdout.flush()
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
300

301
302
303
304
305
306
307
308
309
310
311
312
def error_block_message(block_id, blocks_info, pp_ti):
	bi=blocks_info[str(block_id)]
	comment=bi.get('comment', '')
	C=pp_ti['text'][bi['pos']]
	path=pp_ti['paths'][str(C[0])]
	#TODO handle IOError
	line=open(path).read().split('\n')[C[1]]
	pos=C[2]
	return 'error in block %d in %s:%d:%d:\n%s\n%s' % (block_id,
		path, C[1]+1, C[2]+1, comment,
		mark(line, pos))

313
def so_build_message(eln, epos, emsg):
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
314
	#emsg=emsg.decode('utf-8')
315
316
317
318
319
320
321
322
	ln=1
	cur=1
	for i in range(len(conf['fb.ti']['text'])):
		assert ln<=eln and (cur<=epos or ln<eln)

		if ln==eln and cur==epos:
			C=conf['fb.ti']['text'][i]
			mrk=mark(conf['fb.cpp'][ln-1], cur-1)
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
323
			return 'compile: at %s:%d:%d: %s\n%s' % (
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
				conf['fb.ti']['paths'][str(C[0])], C[1], C[2],
				emsg, mrk)
		if conf['fb.ti']['text'][i][3]=='\n':
			ln+=1
			cur=1
		else:
			cur+=1
	raise Exception('reached eof while seeking for error pos')
	
def get_src(path):
	return open(path).read().split('\n')

def translate_so_build_err(err):
	err=err.split('\n')
	cur=0

	res=[]

	while cur<len(err):
		ln=err[cur]
		if ln.startswith(conf['BUILD_DIR']):
			m=re.match('%s:([0-9]+):([0-9]+): (.*)$' % os.path.join(
				conf['BUILD_DIR'], 'foreign_blocks.cpp'), ln)
			m2=re.match('%s:.*void __foreign_block_([0-9]+)\(.*' \
				% os.path.join(
				conf['BUILD_DIR'], 'foreign_blocks.cpp'), ln)
			if m is not None:
				res.append(so_build_message(int(m.groups()[0]),
					int(m.groups()[1]), m.groups()[2]))
			elif m2 is not None:
				bid=int(m2.groups()[0])
				C=conf['blocks.ti']['text'][bid][0]
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
356
				res.append('in foreign block at %s:%d:%d:\n%s' % (
357
358
359
					conf['blocks.ti']['paths'][str(C[0])],C[1], C[2],
					mark(get_src(conf['blocks.ti']['paths'][str(C[0])])[C[1]], C[2])))
			else:
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
360
				res.append(ln)#.decode('utf-8'))
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
361
		else:
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
362
			res.append(ln)#.decode('utf-8'))
363
364
365

		cur+=1
	
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
366
	return '\n'.join(res)
367

Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388

def translate_fcmp2_err(err):
	res=[]

	for ln in err.split('\n'):
		m=re.match('#ERROR at ([0-9]+): (.*)$', ln)
		if m is None:
			res.append('err> %s' % ln)
		else:
			pos=int(m.groups()[0])
			C=ti['text'][pos]
			path=ti['paths'][str(C[0])]
			ln=C[1]
			ln_pos=C[2]
			line=get_src(ti['paths'][str(C[0])])[ln]
			mmsg=mark(line, ln_pos)

			res.append('%s at %s:%d:%d:\n%s' % (
				m.groups()[1], path, ln+1, ln_pos+1, mmsg))
	return '\n'.join(res)

Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
389
def main():
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
390
	global conf, ti
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
391
	if '--help' in sys.argv[1:]:
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
392
		print(HELP_MESSAGE)
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
393
		sys.exit(0)
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
394

Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
395
396
397
	conf=parse_args(sys.argv[1:])
	parse_env(conf)

398
	# Set unset parameters to default values
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
399
	if 'BUILD_DIR' not in conf:
400
401
402
403
404
405
		path=os.path.abspath(conf['PROGRAM'])
		dirs=[]
		while os.path.split(path)[1]:
			path, tail=os.path.split(path)
			dirs=[tail]+dirs
		conf['BUILD_DIR']=os.path.join(conf['LUNA_HOME'], 'build', 'programs', *dirs)
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
406

Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
407
	if 'PYTHON' not in conf:
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
408
		conf['PYTHON']='python3'
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
409
410
411

	conf['CXX_FLAGS']=conf.get('CXX_FLAGS', '') \
		+ ' -I %s' % os.path.join(conf['LUNA_HOME'], 'include') \
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
412
		+ ' -std=c++11 -fPIC -Wall -Wpedantic -Wno-vla' \
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
413
414
415
		+ ' -Wno-sign-compare -Wno-unused-but-set-variable' \
		+ ' -Wno-unused-variable'

Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
416
417
418
419
420
421
422
423
424
425
426
	if conf['DEBUG']:
		conf['CXX_FLAGS']+=' -g -DADD_DEBUG_INFO'
	
	if conf['OPTIMIZATION']=='-O':
		conf['CXX_FLAGS']+=' -O3'
	elif conf['OPTIMIZATION']=='-O0':
		conf['CXX_FLAGS']+=' -O0'
	else:
		raise NotImplementedError(conf['OPTIMIZATION'])
		

Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
427
	conf['LDFLAGS']=conf.get('LDFLAGS', '') \
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
428
		+ ' -ldl -L %s' % os.path.join(conf['LUNA_HOME'], 'lib')
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
429
430
	
	if conf['DEBUG']:
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
431
432
433
		conf['LDFLAGS']+=' -g -lrts_dbg'
	else:
		conf['LDFLAGS']+=' -lrts'
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
434
435
	
	# ensure build dir exists
436
	info('creating build dir')
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
437
438
439
440
441
442
443
444
445

	if not os.path.exists(conf['BUILD_DIR']):
		os.makedirs(conf['BUILD_DIR'])
	
	if not os.path.isdir(conf['BUILD_DIR']):
		raise FatalError('not a directory at build dir: %s' \
			% conf['BUILD_DIR'])
	try:

Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
446
		# ==========
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
447
		# preprocess
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
448
		# ==========
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
449

450
451
452
453
454
455
456
457
458
459
		info('preprocessing')
		cmd=[conf['PYTHON']]
		cmd+=[os.path.join(conf['LUNA_HOME'], 'scripts', 'pp.py')]
		cmd+=[conf['PROGRAM']]
		cmd+=['-o', os.path.join(conf['BUILD_DIR'], 'preprocessed.fa')]
		cmd+=['--text-info=%s' % os.path.join(conf['BUILD_DIR'],
			'preprocessed.fa.ti')]
		cmd+=['--blocks-path=%s' % os.path.join(conf['BUILD_DIR'],
			'blocks.ti')]

460
		if VERBOSE_FLAG:
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
461
			print(' '.join(cmd))
462
		p=subprocess.Popen(cmd, stderr=subprocess.PIPE,
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
463
			stdin=subprocess.PIPE, stdout=subprocess.PIPE)
464
		out, err=p.communicate()
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
465
466
		out=out.decode('utf-8')
		err=err.decode('utf-8')
467

468
		if VERBOSE_FLAG:
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
469
			print(out)
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
470

471
472
473
474
475
		if p.returncode!=0:
			raise FatalError('preprocessing failed (see error messages ' \
				+ 'below):\n%s' % err)
		ti=json.loads(open(os.path.join(conf['BUILD_DIR'],
			'preprocessed.fa.ti')).read())
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
476
		
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
477

Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
478
		# =====
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
479
		# parse
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
480
		# =====
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
481

482
		info('parsing')
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
483
484
485
		cmd=[os.path.join(conf['LUNA_HOME'], 'bin', 'parser')]
		cmd+=[os.path.join(conf['BUILD_DIR'], 'preprocessed.fa')]
		cmd+=['-o', os.path.join(conf['BUILD_DIR'], 'program.ja')]
486
		cmd+=['-h', os.path.join(conf['BUILD_DIR'], 'headers.ja')]
487
		if VERBOSE_FLAG:
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
488
			print(' '.join(cmd))
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
489
490
		p=subprocess.Popen(cmd, stderr=subprocess.PIPE, stdout=subprocess.PIPE)
		out, err=p.communicate()
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
491
492
		out=out.decode('utf-8')
		err=err.decode('utf-8')
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
493
494
495
496
497
498
499
500
501
502
503

		if p.returncode!=0:
			for ln in err.split('\n'):
				m=re.match('^#ERROR: ([0-9]+): (.*)$', ln)

				if m is not None:
					pos=int(m.groups()[0])
					raise CompileError(m.groups()[1], ti, pos)
			raise FatalError('compilation failed: ' \
				+ 'see error messages below:\n%s' % err)

504
		if VERBOSE_FLAG:
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
505
			print(out)
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
506

Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
507
		# ================
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
508
		# let substitution
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
509
		# ================
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
510
511
512
513
514
515

		info('let substitution')
		cmd=[conf['PYTHON']]
		cmd+=[os.path.join(conf['LUNA_HOME'], 'scripts',
			'substitution_let.py')]
		cmd+=[os.path.join(conf['BUILD_DIR'], 'program.ja')]
516
		if VERBOSE_FLAG:
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
517
			print(' '.join(cmd))
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
518
519
520
		p=subprocess.Popen(cmd, stderr=subprocess.PIPE,
			stdout=subprocess.PIPE)
		out, err=p.communicate()
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
521
522
		out=out.decode('utf-8')
		err=err.decode('utf-8')
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
523
524
525
526
527

		if p.returncode!=0:
			raise FatalError('compilation failed: ' \
				+ 'see error messages below:\n%s' % err)

528
		if VERBOSE_FLAG:
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
529
			print(out)
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
530

Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
531
532
533
534
		# ===================
		# generate cpp blocks
		# ===================

535
536
		info('generating cpp blocks')

Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
537
538
		generate_cpp_blocks(
			os.path.join(conf['BUILD_DIR'], 'program.ja'),
539
			os.path.join(conf['BUILD_DIR'], 'headers.ja'),
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
540
541
542
543
544
545
546
			os.path.join(conf['BUILD_DIR'], 'preprocessed.fa.ti'),
			os.path.join(conf['BUILD_DIR'], 'blocks.ti'),
			os.path.join(conf['BUILD_DIR'], 'program_foreign.ja'),
			os.path.join(conf['BUILD_DIR'], 'foreign_blocks.cpp'),
			os.path.join(conf['BUILD_DIR'], 'foreign_blocks.cpp.ti')
		)

547
548
549
550
551
552
553
554
555
556
		conf['blocks.ti']=json.loads(open(
			os.path.join(conf['BUILD_DIR'], 'blocks.ti')
			).read())
		conf['fb.ti']=json.loads(open(
			os.path.join(conf['BUILD_DIR'], 'foreign_blocks.cpp.ti')
			).read())
		conf['fb.cpp']=open(
			os.path.join(conf['BUILD_DIR'], 'foreign_blocks.cpp')).read() \
				.split('\n')

Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
557
558
559
560
561

		# ===============
		# generate recoms
		# ===============

562
		info('generating recommendations')
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
563
564
565
566
567
568

		cmd=[conf['PYTHON'],
			os.path.join(conf['LUNA_HOME'], 'scripts', 'fcmp'),
			os.path.join(conf['BUILD_DIR'], 'program_foreign.ja'),
			os.path.join(conf['BUILD_DIR'], 'program_recom.ja'),
			'--only-requests']
569
		if VERBOSE_FLAG:
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
570
			print(' '.join(cmd))
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
571
572
		p=subprocess.Popen(cmd, stderr=subprocess.PIPE, stdout=subprocess.PIPE)
		out, err=p.communicate()
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
573
574
		out=out.decode('utf-8')
		err=err.decode('utf-8')
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
575
576
577
578
579

		if p.returncode!=0:
			raise FatalError('recom-generation failed (see below):\n%s' \
				% err)

580
		if VERBOSE_FLAG:
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
581
			print(out)
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
582

Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
583
		# ================
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
584
		# Generate ja->cpp
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
585
		# ================
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
586

587
588
		info('generating cpp code')

Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
589
590
591
		cmd=[conf['PYTHON'],
			os.path.join(conf['LUNA_HOME'], 'scripts', 'fcmp2'),
			os.path.join(conf['BUILD_DIR'], 'program_recom.ja'),
592
			os.path.join(conf['BUILD_DIR'], 'test.cpp'),
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
593
			os.path.join(conf['BUILD_DIR'], 'cpp_blocks_info.json'),
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
594
595
		]

Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
596
597
598
		if conf['DEBUG']:
			cmd.append('--add-debug-info')

599
		if VERBOSE_FLAG:
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
600
			print(' '.join(cmd))
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
601
602
		p=subprocess.Popen(cmd, stderr=subprocess.PIPE, stdout=subprocess.PIPE)
		out, err=p.communicate()
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
603
604
		out=out.decode('utf-8')
		err=err.decode('utf-8')
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
605
606
607

		if p.returncode!=0:
			raise FatalError('cpp-generation failed (see below):\n%s' \
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
608
				% translate_fcmp2_err(err))
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
609
		
610
		if VERBOSE_FLAG:
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
611
			print(out)
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
612

613
614
		cpp_blocks_info=json.loads(open(
			os.path.join(conf['BUILD_DIR'], 'cpp_blocks_info.json')).read())
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
615

Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
616
		# =======================
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
617
		# build makefile for *.so
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
618
619
		# =======================

620
		info('constructing makefile')
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
621

Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
622
623
624
625
626
627
628
629
630
		cmd=[conf['PYTHON'],
			os.path.join(conf['LUNA_HOME'], 'scripts', 'mkgen'),
			'--out=%s' % os.path.join(conf['BUILD_DIR'], 'libucodes.so'),
			'--compile-flags=%s' % conf['CXX_FLAGS'],
			'--compiler=%s' % conf['CXX'],
			'--obj=%s' % conf['BUILD_DIR'],
			'--src-stdin',
			'--link-flags=%s -shared -fPIC' % conf['LDFLAGS']
		]
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
631

632
		if VERBOSE_FLAG:
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
633
			print(' '.join(cmd))
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
634
635
		p=subprocess.Popen(cmd, stderr=subprocess.PIPE, stdout=subprocess.PIPE,
			stdin=subprocess.PIPE)
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
636

Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
637
638
		cpp_list=[os.path.join(conf['PROGRAM_DIR'], f) \
			for f in os.listdir(conf['PROGRAM_DIR']) if f.endswith('.cpp')]
Vladislav Perepelkin's avatar
bug fix    
Vladislav Perepelkin committed
639
		cpp_list+=[os.path.join(conf['BUILD_DIR'], f) \
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
640
			for f in os.listdir(conf['BUILD_DIR']) if f.endswith('.cpp')]
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
641
642
643
		out, err=p.communicate(input=('\n'.join(cpp_list)).encode('utf-8'))
		out=out.decode('utf-8')
		err=err.decode('utf-8')
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
644

Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
645
646
647
		if p.returncode!=0:
			raise FatalError('makefile-generation failed (see below):\n%s' \
				% err)
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
648

Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
649
650
		f=open(os.path.join(conf['BUILD_DIR'], 'Makefile.libucodes'),
			'w').write(out)
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
651

Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
652
		
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
653
		# ========
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
654
		# build so
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
655
		# ========
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
656

657
658
		info('building program')

Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
659
		cmd=['make', '-j8', '-f',
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
660
			os.path.join(conf['BUILD_DIR'], 'Makefile.libucodes')]
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
661

662
		if VERBOSE_FLAG:
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
663
			print(' '.join(cmd))
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
664
665
666
			p=subprocess.Popen(cmd, stderr=subprocess.PIPE)
		else:
			p=subprocess.Popen(cmd, stderr=subprocess.PIPE, stdout=subprocess.PIPE)
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
667
		out, err=p.communicate()
668
669
		out=out.decode('utf-8') if out is not None else ''
		err=err.decode('utf-8') if err is not None else ''
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
670
671

		if p.returncode!=0:
672
			err=translate_so_build_err(err)
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
673
674
			raise FatalError('so-build failed: %d (see below):\n%s' \
				% (p.returncode, err))
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
675

676
		if VERBOSE_FLAG:
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
677
			print(out)
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
678
		
679
		info('running program')
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
680
		info(None)
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
681
682
683

		
		# =======
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
684
		# run rts
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
685
686
		# =======

Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
687
688
		rts='rts.dbg' if conf['DEBUG'] else 'rts'
		cmd=[os.path.join(conf['LUNA_HOME'], 'bin', rts),
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
689
			os.path.join(conf['BUILD_DIR'], 'libucodes.so')] + conf['ARGV']
690
		if conf['BALANCE']:
a-chmil's avatar
a-chmil committed
691
			cmd+='-b'
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
692
693
694
		env=dict(os.environ)
		env['LD_LIBRARY_PATH']=env.get('LD_LIBRARY_PATH', '') \
			+ ':' + os.path.join(conf['LUNA_HOME'], 'lib')
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
695
		
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
696

Anton Kudinov's avatar
Anton Kudinov committed
697
		if VERBOSE_FLAG or (COMPILE_ONLY_FLAG and not QUIET_FLAG):
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
698
			print(' '.join(cmd))
Anton Kudinov's avatar
Anton Kudinov committed
699
700
		if COMPILE_ONLY_FLAG:
			exit(0)
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
701
		t0=datetime.datetime.now()
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
702
		p=subprocess.Popen(cmd, stderr=subprocess.PIPE, env=env)
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
703
704
705
706
707

		# Disable KeyboardInterrupt to allow rts handle the signal
		old_handler=signal.getsignal(signal.SIGINT)
		signal.signal(signal.SIGINT, signal.SIG_IGN)

708
		out, err=p.communicate()
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
709
710
711
712
		if out is not None:
			out=out.decode('utf-8')
		if err is not None:
			err=err.decode('utf-8')
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
713
714
715

		signal.signal(signal.SIGINT, old_handler)

Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
716
		t=datetime.datetime.now()
717
718
719
		if p.returncode!=0:
			msgs=[]
			for ln in err.split('\n'):
720
				recognized_flag=False
721
722
				m=re.match('^#ERROR_BLOCK: ([0-9]+)', ln)
				if m is not None:
723
					recognized_flag=True
724
725
					msgs.append(error_block_message(int(m.groups()[0]),
						cpp_blocks_info, ti))
726
727
728
729
730
731
				m=re.match('^#ERROR_MSG: (.*)$', ln)
				if m is not None:
					recognized_flag=True
					msgs.append('runtime error message: %s' % repr(
						m.groups()[0]))
				if not recognized_flag:
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
732
					msgs.append('err> %s' % ln)
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
733
734
			raise FatalError('run-time error: errcode=%d' % p.returncode \
				+ ''.join(['\n%s' % msg for msg in msgs]))
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
735
736

		if conf['TIME']:
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
737
			print("TIME: %lf sec." % (t-t0).total_seconds())
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
738
739
740
		
	finally:
		if conf['CLEANUP']:
741
			info('cleaning up')
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
742
			try:
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
743
				cmd=['rm', '-r', conf['BUILD_DIR']]
744
				if VERBOSE_FLAG:
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
745
					print(' '.join(cmd))
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
746
				subprocess.check_call(cmd)
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
747
748
			except subprocess.CalledProcessError as x:
				raise FatalError('cleanup failed (errcode=%d)' % x.returncode)
749
		info(None)
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
750
751
752
753
754
755
	
		
if __name__=='__main__':
	try:
		main()
	except FatalError as x:
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
756
		sys.stderr.write(('%s: fatal error: %s\n' % (_argv0, x.message)))
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
757
		sys.exit(1)
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
758
	except CompileError as x:
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
759
		sys.stderr.write('%s: compile error: %s at %s:%d\n' % (
760
			_argv0, x.message, x.path, x.line_num+1))
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
761
		sys.stderr.write(mark(x.line, x.line_pos))
Vladislav Perepelkin's avatar
Vladislav Perepelkin committed
762
		sys.exit(1)