Remove trailing whitespace.
[cython.git] / Cython / Compiler / Code.py
1 # cython: language_level = 3
2 #
3 #   Pyrex - Code output module
4 #
5
6 import cython
7 cython.declare(re=object, Naming=object, Options=object, StringEncoding=object,
8                Utils=object, SourceDescriptor=object, StringIOTree=object,
9                DebugFlags=object, none_or_sub=object)
10
11 import re
12 import Naming
13 import Options
14 import StringEncoding
15 from Cython import Utils
16 from Scanning import SourceDescriptor
17 from Cython.StringIOTree import StringIOTree
18 import DebugFlags
19
20 from Cython.Utils import none_or_sub
21 try:
22     basestring
23 except NameError:
24     basestring = str
25
26 class UtilityCode(object):
27     # Stores utility code to add during code generation.
28     #
29     # See GlobalState.put_utility_code.
30     #
31     # hashes/equals by instance
32
33     def __init__(self, proto=None, impl=None, init=None, cleanup=None, requires=None,
34                  proto_block='utility_code_proto'):
35         # proto_block: Which code block to dump prototype in. See GlobalState.
36         self.proto = proto
37         self.impl = impl
38         self.init = init
39         self.cleanup = cleanup
40         self.requires = requires
41         self._cache = {}
42         self.specialize_list = []
43         self.proto_block = proto_block
44
45     def specialize(self, pyrex_type=None, **data):
46         # Dicts aren't hashable...
47         if pyrex_type is not None:
48             data['type'] = pyrex_type.declaration_code('')
49             data['type_name'] = pyrex_type.specialization_name()
50         key = data.items(); key.sort(); key = tuple(key)
51         try:
52             return self._cache[key]
53         except KeyError:
54             if self.requires is None:
55                 requires = None
56             else:
57                 requires = [r.specialize(data) for r in self.requires]
58             s = self._cache[key] = UtilityCode(
59                                         none_or_sub(self.proto, data),
60                                         none_or_sub(self.impl, data),
61                                         none_or_sub(self.init, data),
62                                         none_or_sub(self.cleanup, data),
63                                         requires, self.proto_block)
64             self.specialize_list.append(s)
65             return s
66
67     def put_code(self, output):
68         if self.requires:
69             for dependency in self.requires:
70                 output.use_utility_code(dependency)
71         if self.proto:
72             output[self.proto_block].put(self.proto)
73         if self.impl:
74             output['utility_code_def'].put(self.impl)
75         if self.init:
76             writer = output['init_globals']
77             if isinstance(self.init, basestring):
78                 writer.put(self.init)
79             else:
80                 self.init(writer, output.module_pos)
81         if self.cleanup and Options.generate_cleanup_code:
82             writer = output['cleanup_globals']
83             if isinstance(self.cleanup, basestring):
84                 writer.put(self.cleanup)
85             else:
86                 self.cleanup(writer, output.module_pos)
87
88
89
90 class FunctionState(object):
91     # return_label     string          function return point label
92     # error_label      string          error catch point label
93     # continue_label   string          loop continue point label
94     # break_label      string          loop break point label
95     # return_from_error_cleanup_label string
96     # label_counter    integer         counter for naming labels
97     # in_try_finally   boolean         inside try of try...finally
98     # exc_vars         (string * 3)    exception variables for reraise, or None
99
100     # Not used for now, perhaps later
101     def __init__(self, owner, names_taken=cython.set()):
102         self.names_taken = names_taken
103         self.owner = owner
104
105         self.error_label = None
106         self.label_counter = 0
107         self.labels_used = cython.set()
108         self.return_label = self.new_label()
109         self.new_error_label()
110         self.continue_label = None
111         self.break_label = None
112
113         self.in_try_finally = 0
114         self.exc_vars = None
115
116         self.temps_allocated = [] # of (name, type, manage_ref)
117         self.temps_free = {} # (type, manage_ref) -> list of free vars with same type/managed status
118         self.temps_used_type = {} # name -> (type, manage_ref)
119         self.temp_counter = 0
120
121     # labels
122
123     def new_label(self, name=None):
124         n = self.label_counter
125         self.label_counter = n + 1
126         label = "%s%d" % (Naming.label_prefix, n)
127         if name is not None:
128             label += '_' + name
129         return label
130
131     def new_error_label(self):
132         old_err_lbl = self.error_label
133         self.error_label = self.new_label('error')
134         return old_err_lbl
135
136     def get_loop_labels(self):
137         return (
138             self.continue_label,
139             self.break_label)
140
141     def set_loop_labels(self, labels):
142         (self.continue_label,
143          self.break_label) = labels
144
145     def new_loop_labels(self):
146         old_labels = self.get_loop_labels()
147         self.set_loop_labels(
148             (self.new_label("continue"),
149              self.new_label("break")))
150         return old_labels
151
152     def get_all_labels(self):
153         return (
154             self.continue_label,
155             self.break_label,
156             self.return_label,
157             self.error_label)
158
159     def set_all_labels(self, labels):
160         (self.continue_label,
161          self.break_label,
162          self.return_label,
163          self.error_label) = labels
164
165     def all_new_labels(self):
166         old_labels = self.get_all_labels()
167         new_labels = []
168         for old_label in old_labels:
169             if old_label:
170                 new_labels.append(self.new_label())
171             else:
172                 new_labels.append(old_label)
173         self.set_all_labels(new_labels)
174         return old_labels
175
176     def use_label(self, lbl):
177         self.labels_used.add(lbl)
178
179     def label_used(self, lbl):
180         return lbl in self.labels_used
181
182     # temp handling
183
184     def allocate_temp(self, type, manage_ref):
185         """
186         Allocates a temporary (which may create a new one or get a previously
187         allocated and released one of the same type). Type is simply registered
188         and handed back, but will usually be a PyrexType.
189
190         If type.is_pyobject, manage_ref comes into play. If manage_ref is set to
191         True, the temp will be decref-ed on return statements and in exception
192         handling clauses. Otherwise the caller has to deal with any reference
193         counting of the variable.
194
195         If not type.is_pyobject, then manage_ref will be ignored, but it
196         still has to be passed. It is recommended to pass False by convention
197         if it is known that type will never be a Python object.
198
199         A C string referring to the variable is returned.
200         """
201         if not type.is_pyobject:
202             # Make manage_ref canonical, so that manage_ref will always mean
203             # a decref is needed.
204             manage_ref = False
205         freelist = self.temps_free.get((type, manage_ref))
206         if freelist is not None and len(freelist) > 0:
207             result = freelist.pop()
208         else:
209             while True:
210                 self.temp_counter += 1
211                 result = "%s%d" % (Naming.codewriter_temp_prefix, self.temp_counter)
212                 if not result in self.names_taken: break
213             self.temps_allocated.append((result, type, manage_ref))
214         self.temps_used_type[result] = (type, manage_ref)
215         if DebugFlags.debug_temp_code_comments:
216             self.owner.putln("/* %s allocated */" % result)
217         return result
218
219     def release_temp(self, name):
220         """
221         Releases a temporary so that it can be reused by other code needing
222         a temp of the same type.
223         """
224         type, manage_ref = self.temps_used_type[name]
225         freelist = self.temps_free.get((type, manage_ref))
226         if freelist is None:
227             freelist = []
228             self.temps_free[(type, manage_ref)] = freelist
229         if name in freelist:
230             raise RuntimeError("Temp %s freed twice!" % name)
231         freelist.append(name)
232         if DebugFlags.debug_temp_code_comments:
233             self.owner.putln("/* %s released */" % name)
234
235     def temps_in_use(self):
236         """Return a list of (cname,type,manage_ref) tuples of temp names and their type
237         that are currently in use.
238         """
239         used = []
240         for name, type, manage_ref in self.temps_allocated:
241             freelist = self.temps_free.get((type, manage_ref))
242             if freelist is None or name not in freelist:
243                 used.append((name, type, manage_ref))
244         return used
245
246     def temps_holding_reference(self):
247         """Return a list of (cname,type) tuples of temp names and their type
248         that are currently in use. This includes only temps of a
249         Python object type which owns its reference.
250         """
251         return [(name, type)
252                 for name, type, manage_ref in self.temps_in_use()
253                 if manage_ref]
254
255     def all_managed_temps(self):
256         """Return a list of (cname, type) tuples of refcount-managed Python objects.
257         """
258         return [(cname, type)
259                 for cname, type, manage_ref in self.temps_allocated
260                 if manage_ref]
261
262     def all_free_managed_temps(self):
263         """Return a list of (cname, type) tuples of refcount-managed Python
264         objects that are not currently in use.  This is used by
265         try-except and try-finally blocks to clean up temps in the
266         error case.
267         """
268         return [(cname, type)
269                 for (type, manage_ref), freelist in self.temps_free.items()
270                 if manage_ref
271                 for cname in freelist]
272
273
274 class IntConst(object):
275     """Global info about a Python integer constant held by GlobalState.
276     """
277     # cname     string
278     # value     int
279     # is_long   boolean
280
281     def __init__(self, cname, value, is_long):
282         self.cname = cname
283         self.value = value
284         self.is_long = is_long
285
286 class PyObjectConst(object):
287     """Global info about a generic constant held by GlobalState.
288     """
289     # cname       string
290     # type        PyrexType
291
292     def __init__(self, cname, type):
293         self.cname = cname
294         self.type = type
295
296 cython.declare(possible_unicode_identifier=object, possible_bytes_identifier=object,
297                nice_identifier=object, find_alphanums=object)
298 possible_unicode_identifier = re.compile(ur"(?![0-9])\w+$", re.U).match
299 possible_bytes_identifier = re.compile(r"(?![0-9])\w+$".encode('ASCII')).match
300 nice_identifier = re.compile(r'\A[a-zA-Z0-9_]+\Z').match
301 find_alphanums = re.compile('([a-zA-Z0-9]+)').findall
302
303 class StringConst(object):
304     """Global info about a C string constant held by GlobalState.
305     """
306     # cname            string
307     # text             EncodedString or BytesLiteral
308     # py_strings       {(identifier, encoding) : PyStringConst}
309
310     def __init__(self, cname, text, byte_string):
311         self.cname = cname
312         self.text = text
313         self.escaped_value = StringEncoding.escape_byte_string(byte_string)
314         self.py_strings = None
315
316     def get_py_string_const(self, encoding, identifier=None, is_str=False):
317         py_strings = self.py_strings
318         text = self.text
319
320         is_str = bool(identifier or is_str)
321         is_unicode = encoding is None and not is_str
322
323         if encoding is None:
324             # unicode string
325             encoding_key = None
326         else:
327             # bytes or str
328             encoding = encoding.lower()
329             if encoding in ('utf8', 'utf-8', 'ascii', 'usascii', 'us-ascii'):
330                 encoding = None
331                 encoding_key = None
332             else:
333                 encoding_key = ''.join(find_alphanums(encoding))
334
335         key = (is_str, is_unicode, encoding_key)
336         if py_strings is not None and key in py_strings:
337             py_string = py_strings[key]
338         else:
339             if py_strings is None:
340                 self.py_strings = {}
341             if identifier:
342                 intern = True
343             elif identifier is None:
344                 if isinstance(text, unicode):
345                     intern = bool(possible_unicode_identifier(text))
346                 else:
347                     intern = bool(possible_bytes_identifier(text))
348             else:
349                 intern = False
350             if intern:
351                 prefix = Naming.interned_str_prefix
352             else:
353                 prefix = Naming.py_const_prefix
354             pystring_cname = "%s%s_%s" % (
355                 prefix,
356                 (is_str and 's') or (is_unicode and 'u') or 'b',
357                 self.cname[len(Naming.const_prefix):])
358
359             py_string = PyStringConst(
360                 pystring_cname, encoding, is_unicode, is_str, intern)
361             self.py_strings[key] = py_string
362
363         return py_string
364
365 class PyStringConst(object):
366     """Global info about a Python string constant held by GlobalState.
367     """
368     # cname       string
369     # encoding    string
370     # intern      boolean
371     # is_unicode  boolean
372     # is_str      boolean
373
374     def __init__(self, cname, encoding, is_unicode, is_str=False, intern=False):
375         self.cname = cname
376         self.encoding = encoding
377         self.is_str = is_str
378         self.is_unicode = is_unicode
379         self.intern = intern
380
381     def __lt__(self, other):
382         return self.cname < other.cname
383
384
385 class GlobalState(object):
386     # filename_table   {string : int}  for finding filename table indexes
387     # filename_list    [string]        filenames in filename table order
388     # input_file_contents dict         contents (=list of lines) of any file that was used as input
389     #                                  to create this output C code.  This is
390     #                                  used to annotate the comments.
391     #
392     # utility_codes   set                IDs of used utility code (to avoid reinsertion)
393     #
394     # declared_cnames  {string:Entry}  used in a transition phase to merge pxd-declared
395     #                                  constants etc. into the pyx-declared ones (i.e,
396     #                                  check if constants are already added).
397     #                                  In time, hopefully the literals etc. will be
398     #                                  supplied directly instead.
399     #
400     # const_cname_counter int          global counter for constant identifiers
401     #
402
403     # parts            {string:CCodeWriter}
404
405
406     # interned_strings
407     # consts
408     # interned_nums
409
410     # directives       set             Temporary variable used to track
411     #                                  the current set of directives in the code generation
412     #                                  process.
413
414     directives = {}
415
416     code_layout = [
417         'h_code',
418         'filename_table',
419         'utility_code_proto_before_types',
420         'numeric_typedefs',          # Let these detailed individual parts stay!,
421         'complex_type_declarations', # as the proper solution is to make a full DAG...
422         'type_declarations',         # More coarse-grained blocks would simply hide
423         'utility_code_proto',        # the ugliness, not fix it
424         'module_declarations',
425         'typeinfo',
426         'before_global_var',
427         'global_var',
428         'decls',
429         'all_the_rest',
430         'pystring_table',
431         'cached_builtins',
432         'cached_constants',
433         'init_globals',
434         'init_module',
435         'cleanup_globals',
436         'cleanup_module',
437         'main_method',
438         'utility_code_def',
439         'end'
440     ]
441
442
443     def __init__(self, writer, emit_linenums=False):
444         self.filename_table = {}
445         self.filename_list = []
446         self.input_file_contents = {}
447         self.utility_codes = cython.set()
448         self.declared_cnames = {}
449         self.in_utility_code_generation = False
450         self.emit_linenums = emit_linenums
451         self.parts = {}
452
453         self.const_cname_counter = 1
454         self.string_const_index = {}
455         self.int_const_index = {}
456         self.py_constants = []
457
458         assert writer.globalstate is None
459         writer.globalstate = self
460         self.rootwriter = writer
461
462     def initialize_main_c_code(self):
463         rootwriter = self.rootwriter
464         for part in self.code_layout:
465             self.parts[part] = rootwriter.insertion_point()
466
467         if not Options.cache_builtins:
468             del self.parts['cached_builtins']
469         else:
470             w = self.parts['cached_builtins']
471             w.enter_cfunc_scope()
472             w.putln("static int __Pyx_InitCachedBuiltins(void) {")
473
474         w = self.parts['cached_constants']
475         w.enter_cfunc_scope()
476         w.putln("")
477         w.putln("static int __Pyx_InitCachedConstants(void) {")
478         w.put_setup_refcount_context("__Pyx_InitCachedConstants")
479
480         w = self.parts['init_globals']
481         w.enter_cfunc_scope()
482         w.putln("")
483         w.putln("static int __Pyx_InitGlobals(void) {")
484
485         if not Options.generate_cleanup_code:
486             del self.parts['cleanup_globals']
487         else:
488             w = self.parts['cleanup_globals']
489             w.enter_cfunc_scope()
490             w.putln("")
491             w.putln("static void __Pyx_CleanupGlobals(void) {")
492
493         #
494         # utility_code_def
495         #
496         code = self.parts['utility_code_def']
497         if self.emit_linenums:
498             code.write('\n#line 1 "cython_utility"\n')
499         code.putln("")
500         code.putln("/* Runtime support code */")
501
502     def finalize_main_c_code(self):
503         self.close_global_decls()
504
505         #
506         # utility_code_def
507         #
508         code = self.parts['utility_code_def']
509         import PyrexTypes
510         code.put(PyrexTypes.type_conversion_functions)
511         code.putln("")
512
513     def __getitem__(self, key):
514         return self.parts[key]
515
516     #
517     # Global constants, interned objects, etc.
518     #
519     def close_global_decls(self):
520         # This is called when it is known that no more global declarations will
521         # declared.
522         self.generate_const_declarations()
523         if Options.cache_builtins:
524             w = self.parts['cached_builtins']
525             w.putln("return 0;")
526             if w.label_used(w.error_label):
527                 w.put_label(w.error_label)
528                 w.putln("return -1;")
529             w.putln("}")
530             w.exit_cfunc_scope()
531
532         w = self.parts['cached_constants']
533         w.put_finish_refcount_context()
534         w.putln("return 0;")
535         if w.label_used(w.error_label):
536             w.put_label(w.error_label)
537             w.put_finish_refcount_context()
538             w.putln("return -1;")
539         w.putln("}")
540         w.exit_cfunc_scope()
541
542         w = self.parts['init_globals']
543         w.putln("return 0;")
544         if w.label_used(w.error_label):
545             w.put_label(w.error_label)
546             w.putln("return -1;")
547         w.putln("}")
548         w.exit_cfunc_scope()
549
550         if Options.generate_cleanup_code:
551             w = self.parts['cleanup_globals']
552             w.putln("}")
553             w.exit_cfunc_scope()
554
555         if Options.generate_cleanup_code:
556             w = self.parts['cleanup_module']
557             w.putln("}")
558             w.exit_cfunc_scope()
559
560     def put_pyobject_decl(self, entry):
561         self['global_var'].putln("static PyObject *%s;" % entry.cname)
562
563     # constant handling at code generation time
564
565     def get_cached_constants_writer(self):
566         return self.parts['cached_constants']
567
568     def get_int_const(self, str_value, longness=False):
569         longness = bool(longness)
570         try:
571             c = self.int_const_index[(str_value, longness)]
572         except KeyError:
573             c = self.new_int_const(str_value, longness)
574         return c
575
576     def get_py_const(self, type, prefix='', cleanup_level=None):
577         # create a new Python object constant
578         const = self.new_py_const(type, prefix)
579         if cleanup_level is not None \
580                and cleanup_level <= Options.generate_cleanup_code:
581             cleanup_writer = self.parts['cleanup_globals']
582             cleanup_writer.put_xdecref_clear(const.cname, type, nanny=False)
583         return const
584
585     def get_string_const(self, text):
586         # return a C string constant, creating a new one if necessary
587         if text.is_unicode:
588             byte_string = text.utf8encode()
589         else:
590             byte_string = text.byteencode()
591         try:
592             c = self.string_const_index[byte_string]
593         except KeyError:
594             c = self.new_string_const(text, byte_string)
595         return c
596
597     def get_py_string_const(self, text, identifier=None, is_str=False):
598         # return a Python string constant, creating a new one if necessary
599         c_string = self.get_string_const(text)
600         py_string = c_string.get_py_string_const(text.encoding, identifier, is_str)
601         return py_string
602
603     def get_interned_identifier(self, text):
604         return self.get_py_string_const(text, identifier=True)
605
606     def new_string_const(self, text, byte_string):
607         cname = self.new_string_const_cname(byte_string)
608         c = StringConst(cname, text, byte_string)
609         self.string_const_index[byte_string] = c
610         return c
611
612     def new_int_const(self, value, longness):
613         cname = self.new_int_const_cname(value, longness)
614         c = IntConst(cname, value, longness)
615         self.int_const_index[(value, longness)] = c
616         return c
617
618     def new_py_const(self, type, prefix=''):
619         cname = self.new_const_cname(prefix)
620         c = PyObjectConst(cname, type)
621         self.py_constants.append(c)
622         return c
623
624     def new_string_const_cname(self, bytes_value, intern=None):
625         # Create a new globally-unique nice name for a C string constant.
626         try:
627             value = bytes_value.decode('ASCII')
628         except UnicodeError:
629             return self.new_const_cname()
630
631         if len(value) < 20 and nice_identifier(value):
632             return "%s_%s" % (Naming.const_prefix, value)
633         else:
634             return self.new_const_cname()
635
636     def new_int_const_cname(self, value, longness):
637         if longness:
638             value += 'L'
639         cname = "%s%s" % (Naming.interned_num_prefix, value)
640         cname = cname.replace('-', 'neg_').replace('.','_')
641         return cname
642
643     def new_const_cname(self, prefix=''):
644         n = self.const_cname_counter
645         self.const_cname_counter += 1
646         return "%s%s%d" % (Naming.const_prefix, prefix, n)
647
648     def add_cached_builtin_decl(self, entry):
649         if Options.cache_builtins:
650             if self.should_declare(entry.cname, entry):
651                 self.put_pyobject_decl(entry)
652                 w = self.parts['cached_builtins']
653                 if entry.name == 'xrange':
654                     # replaced by range() in Py3
655                     w.putln('#if PY_MAJOR_VERSION >= 3')
656                     self.put_cached_builtin_init(
657                         entry.pos, StringEncoding.EncodedString('range'),
658                         entry.cname)
659                     w.putln('#else')
660                 self.put_cached_builtin_init(
661                     entry.pos, StringEncoding.EncodedString(entry.name),
662                     entry.cname)
663                 if entry.name == 'xrange':
664                     w.putln('#endif')
665
666     def put_cached_builtin_init(self, pos, name, cname):
667         w = self.parts['cached_builtins']
668         interned_cname = self.get_interned_identifier(name).cname
669         from ExprNodes import get_name_interned_utility_code
670         self.use_utility_code(get_name_interned_utility_code)
671         w.putln('%s = __Pyx_GetName(%s, %s); if (!%s) %s' % (
672             cname,
673             Naming.builtins_cname,
674             interned_cname,
675             cname,
676             w.error_goto(pos)))
677
678     def generate_const_declarations(self):
679         self.generate_string_constants()
680         self.generate_int_constants()
681         self.generate_object_constant_decls()
682
683     def generate_object_constant_decls(self):
684         consts = [ (len(c.cname), c.cname, c)
685                    for c in self.py_constants ]
686         consts.sort()
687         decls_writer = self.parts['decls']
688         for _, cname, c in consts:
689             decls_writer.putln(
690                 "static %s;" % c.type.declaration_code(cname))
691
692     def generate_string_constants(self):
693         c_consts = [ (len(c.cname), c.cname, c)
694                      for c in self.string_const_index.values() ]
695         c_consts.sort()
696         py_strings = []
697
698         decls_writer = self.parts['decls']
699         for _, cname, c in c_consts:
700             decls_writer.putln('static char %s[] = "%s";' % (
701                 cname, StringEncoding.split_string_literal(c.escaped_value)))
702             if c.py_strings is not None:
703                 for py_string in c.py_strings.values():
704                     py_strings.append((c.cname, len(py_string.cname), py_string))
705
706         if py_strings:
707             import Nodes
708             self.use_utility_code(Nodes.init_string_tab_utility_code)
709
710             py_strings.sort()
711             w = self.parts['pystring_table']
712             w.putln("")
713             w.putln("static __Pyx_StringTabEntry %s[] = {" %
714                                       Naming.stringtab_cname)
715             for c_cname, _, py_string in py_strings:
716                 if not py_string.is_str or not py_string.encoding or \
717                        py_string.encoding in ('ASCII', 'USASCII', 'US-ASCII',
718                                               'UTF8', 'UTF-8'):
719                     encoding = '0'
720                 else:
721                     encoding = '"%s"' % py_string.encoding.lower()
722
723                 decls_writer.putln(
724                     "static PyObject *%s;" % py_string.cname)
725                 w.putln(
726                     "{&%s, %s, sizeof(%s), %s, %d, %d, %d}," % (
727                     py_string.cname,
728                     c_cname,
729                     c_cname,
730                     encoding,
731                     py_string.is_unicode,
732                     py_string.is_str,
733                     py_string.intern
734                     ))
735             w.putln("{0, 0, 0, 0, 0, 0, 0}")
736             w.putln("};")
737
738             init_globals = self.parts['init_globals']
739             init_globals.putln(
740                 "if (__Pyx_InitStrings(%s) < 0) %s;" % (
741                     Naming.stringtab_cname,
742                     init_globals.error_goto(self.module_pos)))
743
744     def generate_int_constants(self):
745         consts = [ (len(c.value), c.value, c.is_long, c)
746                    for c in self.int_const_index.values() ]
747         consts.sort()
748         decls_writer = self.parts['decls']
749         for _, value, longness, c in consts:
750             cname = c.cname
751             decls_writer.putln("static PyObject *%s;" % cname)
752             if longness:
753                 function = '%s = PyLong_FromString((char *)"%s", 0, 0); %s;'
754             elif Utils.long_literal(value):
755                 function = '%s = PyInt_FromString((char *)"%s", 0, 0); %s;'
756             else:
757                 function = "%s = PyInt_FromLong(%s); %s;"
758             init_globals = self.parts['init_globals']
759             init_globals.putln(function % (
760                 cname,
761                 value,
762                 init_globals.error_goto_if_null(cname, self.module_pos)))
763
764     # The functions below are there in a transition phase only
765     # and will be deprecated. They are called from Nodes.BlockNode.
766     # The copy&paste duplication is intentional in order to be able
767     # to see quickly how BlockNode worked, until this is replaced.
768
769     def should_declare(self, cname, entry):
770         if cname in self.declared_cnames:
771             other = self.declared_cnames[cname]
772             assert str(entry.type) == str(other.type)
773             assert entry.init == other.init
774             return False
775         else:
776             self.declared_cnames[cname] = entry
777             return True
778
779     #
780     # File name state
781     #
782
783     def lookup_filename(self, filename):
784         try:
785             index = self.filename_table[filename]
786         except KeyError:
787             index = len(self.filename_list)
788             self.filename_list.append(filename)
789             self.filename_table[filename] = index
790         return index
791
792     def commented_file_contents(self, source_desc):
793         try:
794             return self.input_file_contents[source_desc]
795         except KeyError:
796             pass
797         source_file = source_desc.get_lines(encoding='ASCII',
798                                             error_handling='ignore')
799         try:
800             F = [u' * ' + line.rstrip().replace(
801                     u'*/', u'*[inserted by cython to avoid comment closer]/'
802                     ).replace(
803                     u'/*', u'/[inserted by cython to avoid comment start]*'
804                     )
805                  for line in source_file]
806         finally:
807             if hasattr(source_file, 'close'):
808                 source_file.close()
809         if not F: F.append(u'')
810         self.input_file_contents[source_desc] = F
811         return F
812
813     #
814     # Utility code state
815     #
816
817     def use_utility_code(self, utility_code):
818         """
819         Adds code to the C file. utility_code should
820         a) implement __eq__/__hash__ for the purpose of knowing whether the same
821            code has already been included
822         b) implement put_code, which takes a globalstate instance
823
824         See UtilityCode.
825         """
826         if utility_code not in self.utility_codes:
827             self.utility_codes.add(utility_code)
828             utility_code.put_code(self)
829
830
831 def funccontext_property(name):
832     try:
833         import operator
834         attribute_of = operator.attrgetter(name)
835     except:
836         def attribute_of(o):
837             return getattr(o, name)
838
839     def get(self):
840         return attribute_of(self.funcstate)
841     def set(self, value):
842         setattr(self.funcstate, name, value)
843     return property(get, set)
844
845
846 class CCodeWriter(object):
847     """
848     Utility class to output C code.
849
850     When creating an insertion point one must care about the state that is
851     kept:
852     - formatting state (level, bol) is cloned and used in insertion points
853       as well
854     - labels, temps, exc_vars: One must construct a scope in which these can
855       exist by calling enter_cfunc_scope/exit_cfunc_scope (these are for
856       sanity checking and forward compatabilty). Created insertion points
857       looses this scope and cannot access it.
858     - marker: Not copied to insertion point
859     - filename_table, filename_list, input_file_contents: All codewriters
860       coming from the same root share the same instances simultaneously.
861     """
862
863     # f                file            output file
864     # buffer           StringIOTree
865
866     # level            int             indentation level
867     # bol              bool            beginning of line?
868     # marker           string          comment to emit before next line
869     # funcstate        FunctionState   contains state local to a C function used for code
870     #                                  generation (labels and temps state etc.)
871     # globalstate      GlobalState     contains state global for a C file (input file info,
872     #                                  utility code, declared constants etc.)
873     # emit_linenums    boolean         whether or not to write #line pragmas
874     #
875     # pyclass_stack    list            used during recursive code generation to pass information
876     #                                  about the current class one is in
877
878     globalstate = None
879
880     def __init__(self, create_from=None, buffer=None, copy_formatting=False, emit_linenums=None):
881         if buffer is None: buffer = StringIOTree()
882         self.buffer = buffer
883         self.marker = None
884         self.last_marker_line = 0
885         self.source_desc = ""
886         self.pyclass_stack = []
887
888         self.funcstate = None
889         self.level = 0
890         self.call_level = 0
891         self.bol = 1
892
893         if create_from is not None:
894             # Use same global state
895             self.globalstate = create_from.globalstate
896             # Clone formatting state
897             if copy_formatting:
898                 self.level = create_from.level
899                 self.bol = create_from.bol
900                 self.call_level = create_from.call_level
901         if emit_linenums is None and self.globalstate:
902             self.emit_linenums = self.globalstate.emit_linenums
903         else:
904             self.emit_linenums = emit_linenums
905
906     def create_new(self, create_from, buffer, copy_formatting):
907         # polymorphic constructor -- very slightly more versatile
908         # than using __class__
909         result = CCodeWriter(create_from, buffer, copy_formatting)
910         return result
911
912     def copyto(self, f):
913         self.buffer.copyto(f)
914
915     def getvalue(self):
916         return self.buffer.getvalue()
917
918     def write(self, s):
919         # also put invalid markers (lineno 0), to indicate that those lines
920         # have no Cython source code correspondence
921         if self.marker is None:
922             cython_lineno = self.last_marker_line
923         else:
924             cython_lineno = self.marker[0]
925
926         self.buffer.markers.extend([cython_lineno] * s.count('\n'))
927         self.buffer.write(s)
928
929     def insertion_point(self):
930         other = self.create_new(create_from=self, buffer=self.buffer.insertion_point(), copy_formatting=True)
931         return other
932
933     def new_writer(self):
934         """
935         Creates a new CCodeWriter connected to the same global state, which
936         can later be inserted using insert.
937         """
938         return CCodeWriter(create_from=self)
939
940     def insert(self, writer):
941         """
942         Inserts the contents of another code writer (created with
943         the same global state) in the current location.
944
945         It is ok to write to the inserted writer also after insertion.
946         """
947         assert writer.globalstate is self.globalstate
948         self.buffer.insert(writer.buffer)
949
950     # Properties delegated to function scope
951     label_counter = funccontext_property("label_counter")
952     return_label = funccontext_property("return_label")
953     error_label = funccontext_property("error_label")
954     labels_used = funccontext_property("labels_used")
955     continue_label = funccontext_property("continue_label")
956     break_label = funccontext_property("break_label")
957     return_from_error_cleanup_label = funccontext_property("return_from_error_cleanup_label")
958
959     # Functions delegated to function scope
960     def new_label(self, name=None):    return self.funcstate.new_label(name)
961     def new_error_label(self):         return self.funcstate.new_error_label()
962     def get_loop_labels(self):         return self.funcstate.get_loop_labels()
963     def set_loop_labels(self, labels): return self.funcstate.set_loop_labels(labels)
964     def new_loop_labels(self):         return self.funcstate.new_loop_labels()
965     def get_all_labels(self):          return self.funcstate.get_all_labels()
966     def set_all_labels(self, labels):  return self.funcstate.set_all_labels(labels)
967     def all_new_labels(self):          return self.funcstate.all_new_labels()
968     def use_label(self, lbl):          return self.funcstate.use_label(lbl)
969     def label_used(self, lbl):         return self.funcstate.label_used(lbl)
970
971
972     def enter_cfunc_scope(self):
973         self.funcstate = FunctionState(self)
974
975     def exit_cfunc_scope(self):
976         self.funcstate = None
977
978     # constant handling
979
980     def get_py_num(self, str_value, longness):
981         return self.globalstate.get_int_const(str_value, longness).cname
982
983     def get_py_const(self, type, prefix='', cleanup_level=None):
984         return self.globalstate.get_py_const(type, prefix, cleanup_level).cname
985
986     def get_string_const(self, text):
987         return self.globalstate.get_string_const(text).cname
988
989     def get_py_string_const(self, text, identifier=None, is_str=False):
990         return self.globalstate.get_py_string_const(text, identifier, is_str).cname
991
992     def get_argument_default_const(self, type):
993         return self.globalstate.get_py_const(type).cname
994
995     def intern(self, text):
996         return self.get_py_string_const(text)
997
998     def intern_identifier(self, text):
999         return self.get_py_string_const(text, identifier=True)
1000
1001     def get_cached_constants_writer(self):
1002         return self.globalstate.get_cached_constants_writer()
1003
1004     # code generation
1005
1006     def putln(self, code = "", safe=False):
1007         if self.marker and self.bol:
1008             self.emit_marker()
1009         if self.emit_linenums and self.last_marker_line != 0:
1010             self.write('\n#line %s "%s"\n' % (self.last_marker_line, self.source_desc))
1011
1012         if code:
1013             if safe:
1014                 self.put_safe(code)
1015             else:
1016                 self.put(code)
1017         self.write("\n");
1018         self.bol = 1
1019
1020     def emit_marker(self):
1021         self.write("\n");
1022         self.indent()
1023         self.write("/* %s */\n" % self.marker[1])
1024         self.last_marker_line = self.marker[0]
1025         self.marker = None
1026
1027     def put_safe(self, code):
1028         # put code, but ignore {}
1029         self.write(code)
1030         self.bol = 0
1031
1032     def put(self, code):
1033         fix_indent = False
1034         if "{" in code:
1035             dl = code.count("{")
1036         else:
1037             dl = 0
1038         if "}" in code:
1039             dl -= code.count("}")
1040             if dl < 0:
1041                 self.level += dl
1042             elif dl == 0 and code[0] == "}":
1043                 # special cases like "} else {" need a temporary dedent
1044                 fix_indent = True
1045                 self.level -= 1
1046         if self.bol:
1047             self.indent()
1048         self.write(code)
1049         self.bol = 0
1050         if dl > 0:
1051             self.level += dl
1052         elif fix_indent:
1053             self.level += 1
1054
1055     def increase_indent(self):
1056         self.level = self.level + 1
1057
1058     def decrease_indent(self):
1059         self.level = self.level - 1
1060
1061     def begin_block(self):
1062         self.putln("{")
1063         self.increase_indent()
1064
1065     def end_block(self):
1066         self.decrease_indent()
1067         self.putln("}")
1068
1069     def indent(self):
1070         self.write("  " * self.level)
1071
1072     def get_py_version_hex(self, pyversion):
1073         return "0x%02X%02X%02X%02X" % (tuple(pyversion) + (0,0,0,0))[:4]
1074
1075     def mark_pos(self, pos):
1076         if pos is None:
1077             return
1078         source_desc, line, col = pos
1079         if self.last_marker_line == line:
1080             return
1081         assert isinstance(source_desc, SourceDescriptor)
1082         contents = self.globalstate.commented_file_contents(source_desc)
1083         lines = contents[max(0,line-3):line] # line numbers start at 1
1084         lines[-1] += u'             # <<<<<<<<<<<<<<'
1085         lines += contents[line:line+2]
1086
1087         marker = u'"%s":%d\n%s\n' % (
1088             source_desc.get_escaped_description(), line, u'\n'.join(lines))
1089         self.marker = (line, marker)
1090         if self.emit_linenums:
1091             self.source_desc = source_desc.get_escaped_description()
1092
1093     def put_label(self, lbl):
1094         if lbl in self.funcstate.labels_used:
1095             self.putln("%s:;" % lbl)
1096
1097     def put_goto(self, lbl):
1098         self.funcstate.use_label(lbl)
1099         self.putln("goto %s;" % lbl)
1100
1101     def put_var_declarations(self, entries, static = 0, dll_linkage = None,
1102             definition = True):
1103         for entry in entries:
1104             if not entry.in_cinclude:
1105                 self.put_var_declaration(entry, static, dll_linkage, definition)
1106
1107     def put_var_declaration(self, entry, static = 0, dll_linkage = None,
1108             definition = True):
1109         #print "Code.put_var_declaration:", entry.name, "definition =", definition ###
1110         if entry.in_closure:
1111             return
1112         visibility = entry.visibility
1113         if visibility == 'private' and not definition:
1114             #print "...private and not definition, skipping" ###
1115             return
1116         if not entry.used and visibility == "private":
1117             #print "not used and private, skipping", entry.cname ###
1118             return
1119         storage_class = ""
1120         if visibility == 'extern':
1121             storage_class = Naming.extern_c_macro
1122         elif visibility == 'public':
1123             if not definition:
1124                 storage_class = Naming.extern_c_macro
1125         elif visibility == 'private':
1126             if static:
1127                 storage_class = "static"
1128         if storage_class:
1129             self.put("%s " % storage_class)
1130         if visibility != 'public':
1131             dll_linkage = None
1132         self.put(entry.type.declaration_code(entry.cname,
1133             dll_linkage = dll_linkage))
1134         if entry.init is not None:
1135             self.put_safe(" = %s" % entry.type.literal_code(entry.init))
1136         self.putln(";")
1137
1138     def put_temp_declarations(self, func_context):
1139         for name, type, manage_ref in func_context.temps_allocated:
1140             decl = type.declaration_code(name)
1141             if type.is_pyobject:
1142                 self.putln("%s = NULL;" % decl)
1143             else:
1144                 self.putln("%s;" % decl)
1145
1146     def put_h_guard(self, guard):
1147         self.putln("#ifndef %s" % guard)
1148         self.putln("#define %s" % guard)
1149
1150     def unlikely(self, cond):
1151         if Options.gcc_branch_hints:
1152             return 'unlikely(%s)' % cond
1153         else:
1154             return cond
1155
1156     # Python objects and reference counting
1157
1158     def entry_as_pyobject(self, entry):
1159         type = entry.type
1160         if (not entry.is_self_arg and not entry.type.is_complete()
1161             or entry.type.is_extension_type):
1162             return "(PyObject *)" + entry.cname
1163         else:
1164             return entry.cname
1165
1166     def as_pyobject(self, cname, type):
1167         from PyrexTypes import py_object_type, typecast
1168         return typecast(py_object_type, type, cname)
1169
1170     def put_gotref(self, cname):
1171         self.putln("__Pyx_GOTREF(%s);" % cname)
1172
1173     def put_giveref(self, cname):
1174         self.putln("__Pyx_GIVEREF(%s);" % cname)
1175
1176     def put_xgiveref(self, cname):
1177         self.putln("__Pyx_XGIVEREF(%s);" % cname)
1178
1179     def put_xgotref(self, cname):
1180         self.putln("__Pyx_XGOTREF(%s);" % cname)
1181
1182     def put_incref(self, cname, type, nanny=True):
1183         if nanny:
1184             self.putln("__Pyx_INCREF(%s);" % self.as_pyobject(cname, type))
1185         else:
1186             self.putln("Py_INCREF(%s);" % self.as_pyobject(cname, type))
1187
1188     def put_decref(self, cname, type, nanny=True):
1189         if nanny:
1190             self.putln("__Pyx_DECREF(%s);" % self.as_pyobject(cname, type))
1191         else:
1192             self.putln("Py_DECREF(%s);" % self.as_pyobject(cname, type))
1193
1194     def put_var_gotref(self, entry):
1195         if entry.type.is_pyobject:
1196             self.putln("__Pyx_GOTREF(%s);" % self.entry_as_pyobject(entry))
1197
1198     def put_var_giveref(self, entry):
1199         if entry.type.is_pyobject:
1200             self.putln("__Pyx_GIVEREF(%s);" % self.entry_as_pyobject(entry))
1201
1202     def put_var_xgotref(self, entry):
1203         if entry.type.is_pyobject:
1204             self.putln("__Pyx_XGOTREF(%s);" % self.entry_as_pyobject(entry))
1205
1206     def put_var_xgiveref(self, entry):
1207         if entry.type.is_pyobject:
1208             self.putln("__Pyx_XGIVEREF(%s);" % self.entry_as_pyobject(entry))
1209
1210     def put_var_incref(self, entry):
1211         if entry.type.is_pyobject:
1212             self.putln("__Pyx_INCREF(%s);" % self.entry_as_pyobject(entry))
1213
1214     def put_decref_clear(self, cname, type, nanny=True):
1215         from PyrexTypes import py_object_type, typecast
1216         if nanny:
1217             self.putln("__Pyx_DECREF(%s); %s = 0;" % (
1218                 typecast(py_object_type, type, cname), cname))
1219         else:
1220             self.putln("Py_DECREF(%s); %s = 0;" % (
1221                 typecast(py_object_type, type, cname), cname))
1222
1223     def put_xdecref(self, cname, type, nanny=True):
1224         if nanny:
1225             self.putln("__Pyx_XDECREF(%s);" % self.as_pyobject(cname, type))
1226         else:
1227             self.putln("Py_XDECREF(%s);" % self.as_pyobject(cname, type))
1228
1229     def put_xdecref_clear(self, cname, type, nanny=True):
1230         if nanny:
1231             self.putln("__Pyx_XDECREF(%s); %s = 0;" % (
1232                 self.as_pyobject(cname, type), cname))
1233         else:
1234             self.putln("Py_XDECREF(%s); %s = 0;" % (
1235                 self.as_pyobject(cname, type), cname))
1236
1237     def put_var_decref(self, entry):
1238         if entry.type.is_pyobject:
1239             if entry.init_to_none is False:  # FIXME: 0 and False are treated differently???
1240                 self.putln("__Pyx_XDECREF(%s);" % self.entry_as_pyobject(entry))
1241             else:
1242                 self.putln("__Pyx_DECREF(%s);" % self.entry_as_pyobject(entry))
1243
1244     def put_var_decref_clear(self, entry):
1245         if entry.type.is_pyobject:
1246             self.putln("__Pyx_DECREF(%s); %s = 0;" % (
1247                 self.entry_as_pyobject(entry), entry.cname))
1248
1249     def put_var_xdecref(self, entry):
1250         if entry.type.is_pyobject:
1251             self.putln("__Pyx_XDECREF(%s);" % self.entry_as_pyobject(entry))
1252
1253     def put_var_xdecref_clear(self, entry):
1254         if entry.type.is_pyobject:
1255             self.putln("__Pyx_XDECREF(%s); %s = 0;" % (
1256                 self.entry_as_pyobject(entry), entry.cname))
1257
1258     def put_var_decrefs(self, entries, used_only = 0):
1259         for entry in entries:
1260             if not used_only or entry.used:
1261                 if entry.xdecref_cleanup:
1262                     self.put_var_xdecref(entry)
1263                 else:
1264                     self.put_var_decref(entry)
1265
1266     def put_var_xdecrefs(self, entries):
1267         for entry in entries:
1268             self.put_var_xdecref(entry)
1269
1270     def put_var_xdecrefs_clear(self, entries):
1271         for entry in entries:
1272             self.put_var_xdecref_clear(entry)
1273
1274     def put_init_to_py_none(self, cname, type, nanny=True):
1275         from PyrexTypes import py_object_type, typecast
1276         py_none = typecast(type, py_object_type, "Py_None")
1277         if nanny:
1278             self.putln("%s = %s; __Pyx_INCREF(Py_None);" % (cname, py_none))
1279         else:
1280             self.putln("%s = %s; Py_INCREF(Py_None);" % (cname, py_none))
1281
1282     def put_init_var_to_py_none(self, entry, template = "%s", nanny=True):
1283         code = template % entry.cname
1284         #if entry.type.is_extension_type:
1285         #    code = "((PyObject*)%s)" % code
1286         self.put_init_to_py_none(code, entry.type, nanny)
1287
1288     def put_pymethoddef(self, entry, term, allow_skip=True):
1289         if entry.is_special or entry.name == '__getattribute__':
1290             if entry.name not in ['__cinit__', '__dealloc__', '__richcmp__', '__next__', '__getreadbuffer__', '__getwritebuffer__', '__getsegcount__', '__getcharbuffer__', '__getbuffer__', '__releasebuffer__']:
1291                 if entry.name == '__getattr__' and not self.globalstate.directives['fast_getattr']:
1292                     pass
1293                 # Python's typeobject.c will automatically fill in our slot
1294                 # in add_operators() (called by PyType_Ready) with a value
1295                 # that's better than ours.
1296                 elif allow_skip:
1297                     return
1298         from TypeSlots import method_coexist
1299         if entry.doc:
1300             doc_code = entry.doc_cname
1301         else:
1302             doc_code = 0
1303         method_flags = entry.signature.method_flags()
1304         if method_flags:
1305             if entry.is_special:
1306                 method_flags += [method_coexist]
1307             self.putln(
1308                 '{__Pyx_NAMESTR("%s"), (PyCFunction)%s, %s, __Pyx_DOCSTR(%s)}%s' % (
1309                     entry.name,
1310                     entry.func_cname,
1311                     "|".join(method_flags),
1312                     doc_code,
1313                     term))
1314
1315     # error handling
1316
1317     def put_error_if_neg(self, pos, value):
1318 #        return self.putln("if (unlikely(%s < 0)) %s" % (value, self.error_goto(pos)))  # TODO this path is almost _never_ taken, yet this macro makes is slower!
1319         return self.putln("if (%s < 0) %s" % (value, self.error_goto(pos)))
1320
1321     def set_error_info(self, pos):
1322         if Options.c_line_in_traceback:
1323             cinfo = " %s = %s;" % (Naming.clineno_cname, Naming.line_c_macro)
1324         else:
1325             cinfo = ""
1326         return "%s = %s[%s]; %s = %s;%s" % (
1327             Naming.filename_cname,
1328             Naming.filetable_cname,
1329             self.lookup_filename(pos[0]),
1330             Naming.lineno_cname,
1331             pos[1],
1332             cinfo)
1333
1334     def error_goto(self, pos):
1335         lbl = self.funcstate.error_label
1336         self.funcstate.use_label(lbl)
1337         return "{%s goto %s;}" % (
1338             self.set_error_info(pos),
1339             lbl)
1340
1341     def error_goto_if(self, cond, pos):
1342         return "if (%s) %s" % (self.unlikely(cond), self.error_goto(pos))
1343
1344     def error_goto_if_null(self, cname, pos):
1345         return self.error_goto_if("!%s" % cname, pos)
1346
1347     def error_goto_if_neg(self, cname, pos):
1348         return self.error_goto_if("%s < 0" % cname, pos)
1349
1350     def error_goto_if_PyErr(self, pos):
1351         return self.error_goto_if("PyErr_Occurred()", pos)
1352
1353     def lookup_filename(self, filename):
1354         return self.globalstate.lookup_filename(filename)
1355
1356     def put_setup_refcount_context(self, name):
1357         self.putln('__Pyx_RefNannySetupContext("%s");' % name)
1358
1359     def put_finish_refcount_context(self):
1360         self.putln("__Pyx_RefNannyFinishContext();")
1361
1362     def put_trace_declarations(self):
1363         self.putln('__Pyx_TraceDeclarations');
1364
1365     def put_trace_call(self, name, pos):
1366         self.putln('__Pyx_TraceCall("%s", %s[%s], %s);' % (name, Naming.filetable_cname, self.lookup_filename(pos[0]), pos[1]));
1367
1368     def put_trace_exception(self):
1369         self.putln("__Pyx_TraceException();")
1370
1371     def put_trace_return(self, retvalue_cname):
1372         self.putln("__Pyx_TraceReturn(%s);" % retvalue_cname)
1373
1374
1375 class PyrexCodeWriter(object):
1376     # f                file      output file
1377     # level            int       indentation level
1378
1379     def __init__(self, outfile_name):
1380         self.f = Utils.open_new_file(outfile_name)
1381         self.level = 0
1382
1383     def putln(self, code):
1384         self.f.write("%s%s\n" % (" " * self.level, code))
1385
1386     def indent(self):
1387         self.level += 1
1388
1389     def dedent(self):
1390         self.level -= 1
1391