#!/usr/bin/python
"""
cmTemplate 0.3 - Jun 07, 2002

Author: Chris Monson
Email: chris@bouncingchairs.net
HTTP: http://www.bouncingchairs.net (go to the Open Source section)
License: LGPL (see http://www.gnu.org for details on the LGPL)

In-depth documentation:
    Go to http://www.bouncingchairs.net
    Find the open source area, and this module in it.
    The documentation will be nearby.

This module is very fast.  The parsing need only happen once (use something
like FastCGI or one of the various Python Apache modules to get the ability
to leave things in memory between requests) and the template can be realized
multiple times after that.

Quick overview:
    TEMPLATE:
        Var1 is <?= var1 ?>
        
        <?=for v in var2:?>
        v is <?= v ?>

        <?=endfor?>

    PYTHON:
        import cmTemplate
        import sys

        tpl = cmTemplate.Template( template_file_name, path=[dir1, dir2] )
        t = tpl.new_namespace()

        t.var1 = 'hello'
        t.var2 = [1,2,3,4,5]

        t.output(sys.stdout)

    OUTPUT:
        Var1 is hello
        v is 1
        v is 2
        v is 3
        v is 4
        v is 5

It's really that simple!  Of course, you need to know the basic template
syntax, so here are the main things to know:

    Everything between AND INCLUDING <?= and ?> is stripped out and replaced
    with the appropriate values.  Never will template output have those tags
    unless they are preceded by an odd number of backslashes.

    If the ending tag is followed directly by a newline, the newline will
    also be consumed.  This is the same as PHP's behavior, and allows for
    precise formatting of text.

    When you create a new namespace, the variables you set in that namespace
    are seen as global from the perspective of the template.  So, setting
    t.var1 = 'hello' will make the global variable 'var1' available in the
    template.  If there is an <?=echo var1?> in the template, it will simple
    insert 'hello' (without the quotes) in its place.

    Insert the value of an expression into the output:
        <?=echo expression ?>
        <?= expression ?>

    A for loop:
        <?=for varname in expression:?>
        <?=else:?>                      # optional
        <?=endfor?>

    A while loop:
        <?=while expression:?>
        <?=else:?>                      # optional
        <?=endwhile?>

    An if statement:
        <?=if expression:?>
        <?=elif:?>                      # optional
        <?=else:?>                      # optional
        <?=endif?>

    A def:
        <?=def defname( arg1, arg2, ... ):?>
        <?=echo arg1?>,<?= arg2?>
        <?=enddef?>

    Calling a def with arguments:
        <?=call defname( expr1, expr2, ... )?>

    Executing arbitrary Python code (handle with care):
        <?=exec
            global a
            a = 'hi'
        ?>

    Loops can also have break and continue statements:
        <?=break?>
        <?=continue?>

    There are also some special functions that you can call inside a loop:
        for_count()     # Number of items in for_list()
        for_index()     # Current loop index (starts at 0, increments by 1)
        for_list()      # The list of items being looped over
        for_is_first()  # Is this current item the first one in for_list()?
        for_is_last()   # Is this current item the last one in for_list()?

        TEMPLATE:
            <?=for x in "Larry", "Moe", "Curly" :?>
            Person <?=echo for_index()+1?> of <?=echo for_count()?>: <?=echo x?>
            <?=if for_is_first() :?>
             (first one!)
            <?=elif for_is_last() :?>
             (last one!)
            <?=endif?>
            <?=endfor?>

        OUTPUT:
            Person 1 of 3: Larry (first one!)
            Person 2 of 3: Moe
            Person 3 of 3: Curly (last one!)

    Include another template (can be relative or absolute):
        <?=inc 'filename'?>             # can also use double quotes

    Include another file without processing (can be relative or absolute):
        <?=rawinc 'filename'?>

    A comment (can span multiple lines):
        <?=comment The text of your comment goes here ?>
"""

import re
import string
import os
import os.path
import stat

class Handler:
    def handle(self, is_tag, text):
        pass
    def finish(self):
        pass

class Tokenizer:
    def __init__(self, handler):
        # Create some dummy handlers
        self.handler = handler
        self.init()

    def init(self):
        pass

    def process_block(self, block):
        pass

class ETemplate:
    def __init__(self, msg):
        self.msg = msg
    def __str__(self):
        return self.msg

class EUnexpectedEndSym(ETemplate): pass
class EUnexpectedStartSym(ETemplate): pass
class ENoEndSymFound(ETemplate):
    def __init__(self):
        ETemplate.__init__(self,"Unmatched start symbol in template")

class EInvalidIndentation(ETemplate):
    def __init__(self, indentation):
        str = "Invalid indentation: '%s'" % indentation
        ETemplate.__init__(self, str)

class EInvalidTag(ETemplate):
    def __init__(self, name, contents):
        str = "Invalid tag: name='%s', content='%s'" % (name, contents)
        ETemplate.__init__(self, str)

class EInvalidBlockFormat(ETemplate): pass
class EInvalidForTag(ETemplate): pass
class EInvalidWhileTag(ETemplate): pass
class EInvalidDefTag(ETemplate): pass
class EInvalidCallTag(ETemplate): pass
class EInvalidIncTag(ETemplate):
    def __init__(self, contents):
        str = "Invalid inc tag (missing quotes?): %s" % contents
        ETemplate.__init__(self, str)
    
class EInvalidRawIncTag(ETemplate):
    def __init__(self, contents):
        str = "Invalid rawinc tag (missing quotes?): %s" % contents
        ETemplate.__init__(self, str)

class EBadParseStack(ETemplate):
    def __init__(self):
        ETemplate.__init__(self, "Bad Parse Stack")

class ETPLExpected(EBadParseStack): pass
class EIfBlockExpected(EBadParseStack): pass

class EInvalidEmptyBlock(ETemplate): pass
class EInvalidEmptyTag(ETemplate): pass
class EHangingStartTag(ETemplate):
    def __init__(self, tagname):
        str = "Failed to close a block: '%s'" % tagname
        ETemplate.__init__(self, str)

class EHangingEndTag(ETemplate):
    def __init__(self, tagname):
        str = "Found an end tag without a start tag: '%s'" % tagname
        ETemplate.__init__(self, str)

class EInvalidEchoTag(ETemplate): pass

class EAbsoluteFileNotFound(ETemplate):
    def __init__(self, filename):
        str = "Unable to open file '%s'" % (filename,)
        ETemplate.__init__(self, str)
        
class ERelativeFileNotFound(ETemplate):
    def __init__(self, filename, path):
        str = "Unable to open file '%s' in path '%s'" % (
            filename,
            string.join(path, ",")
            )
        ETemplate.__init__(self, str)

class EDefAlreadyExists(ETemplate):
    def __init__(self, defname):
        str = "Attempted to redefine def '%s'" % defname
        ETemplate.__init__(self, str)

class ERecursiveInclusion(ETemplate):
    def __init__(self, includename, parentname):
        str = "Recursive Inclusion Detected: '%s' attempted to include '%s'" % (
            includename, parentname
            )
        ETemplate.__init__(self, str)

# Specific implementations
class Tokenizer_regex(Tokenizer):
    sym_re = re.compile(r"(\\{0,3})(<\?=|\?>\n?)", re.MULTILINE)
    nl_re = re.compile(r"^\n")

    start_len = 3
    end_len = 2

    min_backtrack = 6 # backtrack ? characters if not found in last buffer

    def _is_end_sym_(self, text):
        return text[:2] == '?>';

    def init(self):
        "Called to start off the parsing"
        self.in_buf = ''
        self.in_pos = 0
        self.in_len = 0
        self.out_buf = ''
        self.eat_nl = 0

        # Start out just parsing text (tags are always explicit)
        self.is_in_tag = 0

    def finish(self):
        "Called to finish off the remaining buffer"
        # The only text left here is going to be the end of the file, and
        # it will always be plain text, not a tag (theoretically)
        if self.is_in_tag:
            raise ENoEndSymFound()
        self.out_buf = self.out_buf + self.in_buf[self.in_pos:]
        self.in_buf = ''
        self.in_pos = 0
        self.in_len = 0
        self.handler.handle(self.is_in_tag, self.out_buf)

        # Call the parser's finish method so that it can clean up.
        self.handler.finish()

    def process_block(self, block):
        "Called for each block read from a file or whatever"
        # Add the current block to the buffer
        block_len = len(block)
        self.in_buf = self.in_buf + block
        self.in_len = self.in_len + block_len

        # Search the input buffer for a sym.
        while self.in_pos < self.in_len:
            # Find a sym
            match = self.sym_re.search( self.in_buf, self.in_pos )
            if match:
                # Found a tag.  Get some information about where it was
                # in the string, and whether it was escaped.
                (bs_span, sym_span) = (match.span(1), match.span(2))
                bs_len = bs_span[1] - bs_span[0]
                sym_len = sym_span[1] - sym_span[0]

                is_escaped = bs_len % 2 # odd number

                # Always dump up to what we found.  If we have more than one
                # preceding backslash, we output a backslash (2 and 3
                # backslashes output, 0 and 1 don't)
                dump_to = bs_span[0]
                if bs_len > 1: dump_to = dump_to + 1

                if is_escaped:
                    # If it's escaped, we dump to the dump_to point, and
                    # then we output the symbol that was found.
                    self.out_buf = (
                        self.out_buf +
                        self.in_buf[self.in_pos:dump_to] + 
                        match.group(2)
                        )

                    # Move the position pointer to the point after
                    # the discovered symbol
                    self.in_pos = sym_span[1]

                else:
                    # Not escaped: Check for an errant symbol (a start sym
                    # inside of a tag, or an end tag outside of one)
                    if self._is_end_sym_(match.group(2)):
                        if not self.is_in_tag:
                            raise EUnexpectedEndSym(match.group(2))
                    else:
                        if self.is_in_tag:
                            raise EUnexpectedStartSym(match.group(2))

                    # Valid sym found: dump buffer, call handler, and
                    # switch states
                    self.out_buf = (
                        self.out_buf +
                        self.in_buf[self.in_pos:dump_to]
                        )
                    if self.out_buf:
                        self.handler.handle(self.is_in_tag, self.out_buf)

                    self.in_buf = self.in_buf[sym_span[1]:]
                    self.in_len = len(self.in_buf)
                    self.in_pos = 0
                    self.out_buf = ''

                    self.is_in_tag = not self.is_in_tag

            else:
                # Didn't find a symbol in this buffer segment.

                # Advance to the length minus backtracking.
                # Then the whole thing starts over.
                # This always breaks us out of the loop, since we have seen
                # the whole string.
                self.out_buf = (self.out_buf +
                    self.in_buf[self.in_pos:-self.min_backtrack])
                self.in_pos = self.in_len - self.min_backtrack
                if self.in_pos < 0: self.in_pos = 0
                break

class _NODE_:
    escape_re = re.compile(r'["\\]')
    def escape_text(self, text):
        return self.escape_re.sub(lambda m: '\\%s' % m.group(), text)

class _TPL_(_NODE_):
    def __init__(self):
        self.name = 'tpl'
        self.text = ''
        self.blk = None
        self.tpl = None

    def output_python(self, indent_cur, indent_string):
        # First create the statement for our text
        if self.text:
            str = indent_cur + '__TPL_WRITER.write("""' + \
                self.escape_text(self.text) + '""")\n'
        else:
            str = ''

        # Next print the contained block and template, if they exist
        # Note that we continue with the same indentation.  This is not
        # a code block, so the stuff under us is part of the same code.
        if self.blk:
            str = str + self.blk.output_python( indent_cur, indent_string )

        if self.tpl:
            str = str + self.tpl.output_python( indent_cur, indent_string )

        return str

class _IF_(_NODE_):
    def __init__(self):
        self.name = 'if'
        self.expression = ''
        self.tpl = _TPL_()
        self.default = None # next in the 'if' line

    def output_python(self, indent_cur, indent_string):
        # Return the actual expression

        str = indent_cur + 'if ' + self.expression + ':\n'

        inside_indent = indent_cur + indent_string

        # We always add a 'pass' in case the statement is empty.  It doesn't
        # hurt anything.
        str = str + inside_indent + 'pass\n'

        # Output the contained template (part of this statement - indent it)
        str = str + self.tpl.output_python( inside_indent, indent_string )

        # Output the default, if there is one.  Note that it goes on
        # the same indentation level as this.  It can be either an elif or
        # an else statement.
        if self.default:
            str = str + self.default.output_python( indent_cur, indent_string )

        return str
        
class _ELIF_(_IF_):
    def __init__(self):
        _IF_.__init__(self)
        self.name = 'elif'

    def output_python(self, indent_cur, indent_string):
        # Return the actual expression
        str = indent_cur + 'elif ' + self.expression + ':\n'

        inside_indent = indent_cur + indent_string

        # We always add a 'pass' in case the statement is empty.  It doesn't
        # hurt anything.
        str = str + inside_indent + 'pass\n'

        # Output the contained template (part of this statement - indent it)
        str = str + self.tpl.output_python( inside_indent, indent_string )

        # Output the default, if there is one.  Note that it goes on
        # the same indentation level as this.  It can be either an elif or
        # an else statement.
        if self.default:
            str = str + self.default.output_python( indent_cur, indent_string )

        return str

class _ELSE_(_NODE_):
    def __init__(self):
        self.name = 'else'
        self.tpl = _TPL_()

    def output_python(self, indent_cur, indent_string):
        # Return the actual expression
        str = indent_cur + 'else:\n'

        inside_indent = indent_cur + indent_string

        # We always add a 'pass' in case the statement is empty.  It doesn't
        # hurt anything.
        str = str + inside_indent + 'pass\n'

        return str + self.tpl.output_python( inside_indent, indent_string )

class _FOR_(_NODE_):
    def __init__(self):
        self.name = 'for'
        self.varname = ''
        self.listexpr = ''
        self.tpl = _TPL_()
        self.default = None

    def output_python(self, indent_cur, indent_string):
        str = ""

        # Output loop state-keeping code (for the for_* functions)
        #     [The double-parenthesis are so that list expressions like "1,2,3"
        #     (a tuple) are passed as a single argument to append()]
        str = str + indent_cur + "_for_list.append((%s))\n" % self.listexpr
        str = str + indent_cur + "_for_count.append(len(_for_list[-1]))\n"
        str = str + indent_cur + "_for_index.append(0)\n"

        # output the expression
        str = str + indent_cur +'for %s in %s:\n' % (self.varname,self.listexpr)

        inside_indent = indent_cur + indent_string

        # We always add a 'pass' in case the statement is empty.  It doesn't
        # hurt anything.
        str = str + inside_indent + 'pass\n'

        # output the template if there is one:
        str = str + self.tpl.output_python( inside_indent, indent_string )

        # Increment loop index counter
        str = str + inside_indent + "_for_index[-1] = _for_index[-1] + 1\n"

        # output the default as appropriate:
        if self.default:
            str = str + self.default.output_python( indent_cur, indent_string )

        # We're done with this loop, so forget its state
        str = str + indent_cur + "_for_list.pop()\n"
        str = str + indent_cur + "_for_count.pop()\n"
        str = str + indent_cur + "_for_index.pop()\n"

        return str

class _WHILE_(_NODE_):
    def __init__(self):
        self.name = 'while'
        self.expression = ''
        self.tpl = _TPL_()
        self.default = None

    def output_python(self, indent_cur, indent_string):
        # output the expression
        str = indent_cur + 'while ' + self.expression + ':\n'

        inside_indent = indent_cur + indent_string

        # We always add a 'pass' in case the statement is empty.  It doesn't
        # hurt anything.
        str = str + inside_indent + 'pass\n'

        # output the template if there is one
        str = str + self.tpl.output_python( inside_indent, indent_string )

        # output the default as appropriate:
        if self.default:
            str = str + self.default.output_python( indent_cur, indent_string )

        return str

class _DEF_(_NODE_):
    def __init__(self):
        self.name = 'def'
        self.funcname = ''
        self.args = ()
        self.tpl = _TPL_()

    def output_python(self, indent_cur, indent_string):
        # output the expression
        str = indent_cur + 'def __TPL_FUNC_%s(__TPL_WRITER,%s):\n' % (
            self.funcname,
            string.join(self.args,',')
            )

        inside_indent = indent_cur + indent_string

        # We always add a 'pass' in case the statement is empty.  It doesn't
        # hurt anything.
        str = str + inside_indent + 'pass\n'

        # output the contained template as necessary
        return str + self.tpl.output_python( inside_indent, indent_string )

class _COMMENT_(_NODE_):
    def __init__(self):
        self.name = 'comment'
        self.comment = ''

    def output_python(self, indent_cur, indent_string):
        # do nothing.  We don't output python code for a comment.  Maybe we
        # should, but we don't.
        pass

class _ECHO_(_NODE_):
    def __init__(self):
        self.name = 'echo'
        self.expression = ''

    def output_python(self, indent_cur, indent_string):
        # Simple.  Just return the string setter for the given variable:
        return indent_cur + '__TPL_WRITER.write(str(' + self.expression + '))\n'

class _CALL_(_NODE_):
    def __init__(self):
        self.name = 'call'
        self.funcname = ''
        self.args = ()

    def output_python(self, indent_cur, indent_string):
        # outputs a simple function call
        return indent_cur+'__TPL_FUNC_%s(__TPL_WRITER, %s)\n' % (
            self.funcname,
            string.join(self.args,',')
            )

class _INC_(_NODE_):
    def __init__(self):
        self.name = 'inc'
        self.filename = ''
        # We don't allow include recursion, so the parse tree
        # doesn't ever include a reference to this node.  We have thus
        # avoided circular references.
        self.parse_tree = None

    def output_python(self, indent_cur, indent_string):
        str = indent_cur + '# Include ' + self.filename + '\n'
        str = str + self.parse_tree.output_python(indent_cur, indent_string)
        return str + indent_cur + '# End Include ' + self.filename + '\n'

class _RAWINC_(_NODE_):
    def __init__(self):
        self.name = 'rawinc'
        self.filename = ''

    def output_python(self, indent_cur, indent_string):
        return indent_cur + '__TPL_WRITER.write(__TPL_RAW_CONTENTS[\'' + \
            self.filename + '\'])\n'

class _EXEC_(_NODE_):
    def __init__(self):
        self.name = 'exec'
        self.expression = ''

    def output_python(self, indent_cur, indent_string):
        # since we don't know what indentation style they are going to use,
        # we actually turn this into an exec call.  This makes our life
        # oh, so much simpler.  It completely sidesteps the indentation problem.
        return indent_cur + 'exec("""' + self.escape_text(self.expression) + \
            '""",globals(),locals())\n'

class _BREAK_(_NODE_):
    def __init__(self):
        self.name = 'break'

    def output_python(self, indent_cur, indent_string):
        return indent_cur + 'break'

class _CONTINUE_(_NODE_):
    def __init__(self):
        self.name = 'continue'

    def output_python(self, indent_cur, indent_string):
        return indent_cur + 'continue'

# -----------------------------------------------------------------------------
# -----------------------------------------------------------------------------
class _TemplateFile_:
    """Encapsulates the concept of a template file.  The INC tags, for
    example, use this class to hold their parsed templates."""
    def __init__(self, parent=None, path=None):
        self._syntax_tree = None

        # We need to know who our parent is so that we can do meaningful
        # things with the "root" template file.
        self._parent = parent
        if self._parent:
            self._root = self._parent.root()
        else:
            self._root = None

        # Set up the search path:
        # We always use the current working directory in the path
        # Then we add the path that was passed in.
        # Then we create a list of absolute pathnames (paths can be relative).
        self._path = [os.getcwd(),]
        if path:
            self._path = self._path + list(path)

        self._abspath = map(lambda x: os.path.abspath(x), self._path)
        self._filename = ''
        self._filetime = 0

    def parse( self, filename ):
        self._parse_file_( filename )

    def filename( self ):
        return self._filename

    def filetime( self ):
        return self._filetime

    def parent(self):
        """Returns the parent of this template file.  The parent is the file
        that included it."""
        return self._parent

    def is_root(self):
        """This is the root file if it has no parent."""
        return self._parent is None

    def root(self):
        """The root template is the one that was originally opened.  This
        code can open several included templates, so we keep track of the
        root as a special template file that has all of the defs and the
        list of parsed includes, among other things."""
        if self._root:
            return self._root
        else:
            return self

    def parse_tree(self):
        return self._parse_tree

    def find_file(self, filename):
        """Find the actual file.  It may be an absolute file, or it may
        be somewhere in the path.  We search for it here and return the
        absolute filename."""
        normfile = os.path.normpath(filename)
        if os.path.isabs(normfile):
            # absolute path
            if not os.path.exists(normfile):
                raise EAbsoluteFileNotFound(normfile)
            else:
                return normfile
        else:
            # relative path
            # Here we cycle through the directories in our path, making them
            # absolute.
            # Then we take the absolute paths and join them one at a time
            # with the filename in question.  If we find one that exists,
            # we're good.  Otherwise, we dump out and complain that we
            # couldn't find it.
            fullname = normfile
            for p in self._abspath:
                fullname = os.path.join(p, normfile)
                if os.path.isfile(fullname):
                    return fullname
            else:
                raise ERelativeFileNotFound(normfile, self._abspath)

    def is_file_included( self, abs_filename ):
        """Check the parentage of this template.  If this template or
        any of its ancestors have the same filename as indicated, return
        true.  This is used to make sure that recursive includes do not
        happen.
        
        This works because we never finish parsing a template until
        all of its children are finished.  This means that a template has
        a static relationship with all of its includes.  So, to test
        for recursion, we simply check the parentage."""
        if abs_filename == self._filename:
            # Short circuit.  If this is the file we are looking for, look
            # no further.
            return 1
        elif self.is_root():
            # If we are the root node, then we just checked our own filename
            # and determined that this wasn't it.  Parentage is clean.
            return 0
        else:
            # We are not the root, and this is not the file.  Keep searching.
            return self._parent.is_file_in_parentage( abs_filename )

    def _parse_file_(self, filename):
        """Read a file and parse it, setting up the parse tree."""

        # We set the filename here, because it will be needed by
        # included files (it's our way of checking that we aren't doing
        # recursive includes)
        self._filename = self.find_file( filename )
        self._filetime = os.stat( self._filename )[stat.ST_MTIME]

        # This requires the filename to be absolute and to exist.  It doesn't
        # check the path or anything else.  Hence the private notation.

        # Create a tokenizer and a syntax handler (template parser)
        # Note that we avoid circular references here by making syn_handler
        # a local variable instead of an instance variable.  We create
        # a parser, then we destroy it.
        syn_handler = _TemplateParser_( self )
        tokenizer = Tokenizer_regex( syn_handler )

        # Get the tokenizer going.  When it is finished, the syn_handler
        # will have a syntax tree that we can store and use to generate code.

        tokenizer.init()
        lineno = 0

        f = open(self._filename, 'r')
        while 1:
            line = f.readline()
            lineno = lineno + 1
            if not line: break
            try:
                tokenizer.process_block(line)
            except ETemplate, e:
                print "ERROR (%s:%d): %s" % (filename, lineno, e)
                raise

        f.close()
        tokenizer.finish()

        # We save the only important information from the parser:
        self._parse_tree = syn_handler.parse_tree()

# -----------------------------------------------------------------------------
class _RootTemplateFile_(_TemplateFile_):
    """Encapsulates the extra functionality that the 'root' template
    has.  The root template contains the def table and the include list
    and deals with the support functions that come with a fully-parsed
    template"""
    def __init__(self, path=None):
        _TemplateFile_.__init__(self, parent=None, path=path)

        self._def_list = []
        self._def_dict = {}
        self._includes = {}
        self._raw_includes = {}

        self._indent = ' ' # minimal indentation

    def include_dict( self ):
        """Return a list of included files and their mtimes.
        Note that this is a 'flattened' list, since the includes
        are all listed on the same level.  If an included file includes
        another file, they will both be in this list."""
        # Return a list of includes AND raw includes.
        includes = {}
        for i in self._includes.keys():
            includes[i] = self._includes[i]['time']

        for i in self._raw_includes.keys():
            includes[i] = self._raw_includes[i]['time']

        return includes
            

    def indent_str( self ):
        return self._indent

    def set_indent_str( self, indent_str ):
        # Make sure that the indentation is only whitespace
        if string.strip(indent_str):
            raise EInvalidIndentation( indent_str )

        self._indent_str = indent_str

    def def_exists( self, defname ):
        return self._def_dict.has_key( defname )

    def def_add( self, defname, parsetree ):
        if not self.def_exists( defname ):
            self._def_list.append( defname )
            self._def_dict[ defname ] = parsetree
        # else do nothing.  It's up to the handler to determine what to do
        # if a def name is already taken.

    def def_get( self, defname ):
        return self._defdict[ defname ]

    def include_exists( self, abs_filename ):
        return self._includes.has_key( abs_filename )

    def include_add( self, abs_filename, filetime, parsetree ):
        self._includes[ abs_filename ] = {'time': filetime, 'tree': parsetree}

    def include_get( self, abs_filename ):
        return self._includes[ abs_filename ]

    def raw_exists( self, abs_filename ):
        return self._raw_includes.has_key( abs_filename )

    def raw_add( self, abs_filename, filetime ):
        self._raw_includes[ abs_filename ] = {'time': filetime}

    def _load_raw_contents( self ):
        """Returns a dictionary with the raw filenames as keys and the
        contents as values."""
        raw_contents = {}
        for r in self._raw_includes.keys():
            # update the mtime for this file whenever we read it.
            self._raw_includes[ r ]['time'] = os.stat(r)[stat.ST_MTIME]
            # Now open it, get its contents, and be done with it.
            f = open( r, 'r' )
            raw_contents[r] = f.read()
            f.close()

        return raw_contents

    def output_python( self ):
        """Output the code that can be interpreted by Python.  This includes
        setting up some support stuff, like the string handler and the
        raw include text dictionary."""
        str = ''

        # Add the default string handler
        str = str + """class __TPL_STR_CLASS:
%(in)sdef __init__(self):
%(in2)sself._str = ''
%(in)sdef write(self, s):
%(in2)sself._str = self._str + s
%(in)sdef string(self):
%(in2)sreturn self._str
__TPL_STR = __TPL_STR_CLASS()
""" % {'in': self._indent, 'in2': self._indent * 2}

        # Now we add the raw includes:
        str = str+'__TPL_RAW_CONTENTS = %s\n'%repr(self._load_raw_contents())

        # Now that we have all of the supporting stuff in place, we
        # can print out the code for the defs in our list.
        str = str+'# Beginning def list\n'
        for d in self._def_list:
            str = str + self._def_dict[d].output_python('', self._indent)
        str = str+'# End of def list\n'
        
        str = str + """# Begin for-loop utilities
_for_index = []
_for_count = []
_for_list = []
def for_count(depth=0):     return _for_count[-(depth+1)]
def for_index(depth=0):     return _for_index[-(depth+1)]
def for_list(depth=0):      return _for_list[-(depth+1)]
def for_is_first(depth=0):  return for_index(depth)==0
def for_is_last(depth=0):   return for_index(depth)==for_count(depth)-1
# End for-loop utilities
"""

        # Now we print out the template function header
        str = str+'# Begin main output def\n'
        str = str+'def output(__TPL_WRITER):\n'
        str = str+self.parse_tree().output_python(self._indent, self._indent)
        str = str+'# End main output def\n'
        str = str+'# Begin other utilities\n'
        str = str + \
"""def output_str():
%(in)soutput(__TPL_STR)
%(in)sreturn __TPL_STR.string()
""" % {'in': self._indent}
        str = str+'# End other utilities\n'

        return str

# -----------------------------------------------------------------------------
class _TemplateParser_(Handler):
    """This class handles all of the tags that come in from the tokenizer.
    It determines what to place into the parse tree and when.  It does nothing
    else.  It just deals with the parse tree."""

    tag_re = re.compile(r"^(\w+)(\s+(.*?)\s*)?$", re.MULTILINE)
    empty_block_re = re.compile(r"^(\w+)\s*(:)\s*$", re.MULTILINE)
    if_re = re.compile(r"^(.*):$", re.MULTILINE)
    for_re = re.compile(r"^(\w+)\s+in\s+(.*):$", re.MULTILINE)
    while_re = re.compile(r"^(.*):$", re.MULTILINE)
    def_re = re.compile(r"^(\w+)\s*\((.*)\)\s*:$", re.MULTILINE)
    call_re = re.compile(r"^(\w+)\s*\((.*)\)$", re.MULTILINE)
    inc_re = re.compile(r"^[\"'].*[\"']$", re.MULTILINE)

    def __init__(self, fileobj):
        self.fileobj = fileobj
        # Add a blank TPL block to the tree as the root
        self.tree_root = _TPL_()

        # Push the tpl_file's tpl block onto the stack
        self.node_stack = [self.tree_root]

        # Create a mapping between tag names and methods
        self.tag_methods = {
            'if': self._on_if_,
            'for': self._on_for_,
            'while': self._on_while_,
            'def': self._on_def_,
            'elif': self._on_elif_,
            'else': self._on_else_,
            'endif': self._on_endif_,
            'endfor': self._on_endfor_,
            'endwhile': self._on_endwhile_,
            'enddef': self._on_enddef_,
            'echo': self._on_echo_,
            'comment': self._on_comment_,
            'call': self._on_call_,
            'inc': self._on_inc_,
            'rawinc': self._on_rawinc_,
            'exec': self._on_exec_,
            'break': self._on_break_,
            'continue': self._on_continue_,
            }

    def handle(self, is_tag, text):
        """Handles each piece of the file (text or tags) as it comes in
        from the parser."""
        if is_tag:
            self.handle_tag( text )
        else:
            self.handle_text( text )

    def finish(self):
        """Clean up.  This constitutes popping the elements off
        of the stack.  They should all be template elements.  If any
        of them are not, we didn't close a construct correctly."""
        while not self._stack_empty_():
            name = self._top_node_().name
            if name != 'tpl':
                raise EHangingStartTag( name )
            self._pop_node_()

    def parse_tree(self):
        return self.tree_root

    def _unquote_inc_(self, inc):
        if inc[0] in ('"',"'"):
            if inc[-1] != inc[0]:
                raise EUnmatchedQuotes()
            return inc[1:-1]
            
    def _push_node_(self, tag):
        self.node_stack.append(tag)

    def _top_node_(self):
        return self.node_stack[-1]

    def _top_tpl_(self):
        tpl = self._top_node_()
        if tpl.name != 'tpl':
            raise ETPLExpected()
        return tpl

    def _pop_node_(self):
        self.node_stack.pop()

    def _stack_empty_(self):
        return len(self.node_stack) == 0

    def _insert_atomic_tag_(self, tag):
        """Called when we get tags that are "atomic", meaning that they
        aren't starting or ending tags.  Things like call, inc, etc. are
        atomic tags."""
        # Get the current template off of the stack.
        _cur_tpl_ = self._top_tpl_()

        # Add the tag to the current template node's block part
        # Create a child template for further text and tag handling
        _cur_tpl_.blk = tag
        _cur_tpl_.tpl = _TPL_()

        # Pop off this one and replace it with its child
        # (This one is full and thus no longer relevant)
        self._pop_node_()
        self._push_node_(_cur_tpl_.tpl)

    def _finish_block_(self, blockname, **args):
        """Called every time an ending tag (endfor, endif, etc) is encountered.
        This pops off as many stack elements as possible (until the start tag)
        and sets up the parse tree to accept another piece of text or another
        tag.  Returns the root of the tree for the recently finished
        construct."""
        # We don't actually create a tag here.  We just end our current
        # spot in the tree.  Pop off everything up to and including
        # the parent tag.  Insert a new TPL into the parent node
        # and push that.

        parent_tpl = None
        while not self._stack_empty_():
            node = self._top_node_()
            self._pop_node_() # always pop it off.  We want the parent.
            # If this is the parent of the block, we're done
            if node.name == blockname:
                parent_tpl = self._top_tpl_()
                break

        if not parent_tpl:
            raise EHangingEndTag(blockname)

        # When the above loop ends, node is the beginning tag that
        # corresponds to the end tag we just received.  We save it and
        # return it in case it is needed (it's the root of the tree for
        # the construct that just finished)
        top_tpl = node

        # If we are supposed to remove this node from the parent, we
        # do that here.
        if args.has_key('detach') and args['detach']:
            parent_tpl.blk = None

        # Add a tpl block to this node since it already has a tag in it.
        # The tpl block will handle the next set of stuff we run into.
        parent_tpl.tpl = _TPL_()
        self._push_node_(parent_tpl.tpl)

        return top_tpl

    def handle_tag(self, text):
        # parse out the tag name and call the appropriate parser.
        # If we don't recognize the tag name, or none is given, then
        # recombine and hand it to the echo function.
        match1 = self.tag_re.search(text)
        match2 = self.empty_block_re.search(text)
        if match1:
            name = match1.group(1)
            contents = match1.group(3)
        elif match2:
            name = match2.group(1)
            contents = match2.group(2)
        else:
            name = 'echo'
            contents = text

        # Default tag is an echo
        if not self.tag_methods.has_key(name):
            raise EInvalidTag(name, contents)

        # Call the appropriate handler
        self.tag_methods[name](contents)

    def handle_text(self, text):
        _cur_tpl_ = self._top_tpl_()
        _cur_tpl_.text = _cur_tpl_.text + text
        
    def _on_if_(self, contents):  
        match = self.if_re.search(contents)
        if not match:
            raise EInvalidBlockFormat(contents)

        # Create a new if block for the parse tree.
        newIf = _IF_()
        newIf.expression = match.group(1)

        # Insert this block into the parse tree
        # (the top node -- should be a tpl object)
        curnode = self._top_tpl_()
        curnode.blk = newIf

        # Push this if tag and its template onto the stack
        self._push_node_(newIf)
        self._push_node_(newIf.tpl)

    def _on_elif_(self, contents):
        match = self.if_re.search(contents)
        if not match:
            raise EInvalidBlockFormat(contents)

        # Create a new elif block for the parse tree.
        newElif = _ELIF_()
        newElif.expression = match.group(1)

        # Insert this block into the parse tree -- it should go
        # into the if or elif parent above it (search upwards in the stack)
        parent_node = None
        while not self._stack_empty_():
            node = self._top_node_()
            if node.name in ('if','elif'):
                parent_node = node
                break
            else:
                self._pop_node_()

        if not parent_node:
            raise EIfBlockExpected()

        # Now we have the latest 'if' or 'elif' block.  Add this
        # tag to its default field.  Push this tag and its template.
        parent_node.default = newElif
        self._push_node_(newElif)
        self._push_node_(newElif.tpl)

    def _on_else_(self, contents):
        if contents != ':':
            raise EInvalidEmptyBlock(contents)

        # Create a new else block for the parse tree.
        newElse = _ELSE_()

        # Insert this block into the parse tree -- it should go
        # into the if or elif parent above it (search upwards in the stack)
        parent_node = None
        while not self._stack_empty_():
            node = self._top_node_()
            if node.name in ('if','elif','for','while'):
                parent_node = node
                break
            else:
                self._pop_node_()

        if not parent_node:
            raise EIfBlockExpected()

        # Now we have the parent.
        # Add to the latest 'if', 'elif', 'for', or 'while'.
        # Push node and tpl onto stack.
        parent_node.default = newElse
        self._push_node_(newElse)
        self._push_node_(newElse.tpl)

    def _on_endif_(self, contents):
        if contents:
            raise EInvalidEmptyTag(contents)

        self._finish_block_( 'if' )

    def _on_for_(self, contents):
        match = self.for_re.search(contents)
        if not match:
            raise EInvalidForTag(contents)

        # Create a new for block
        newFor = _FOR_()
        newFor.varname = match.group(1)
        newFor.listexpr = match.group(2)

        # Add to the parse tree
        curnode = self._top_tpl_()
        curnode.blk = newFor

        # Push onto the stack
        self._push_node_(newFor)
        self._push_node_(newFor.tpl)

    def _on_endfor_(self, contents):
        if contents:
            raise EInvalidEmptyTag(contents)

        self._finish_block_( 'for' )

    def _on_while_(self, contents):
        match = self.while_re.search(contents)
        if not match:
            raise EInvalidWhileTag(contents)

        # Create a new while block
        newWhile = _WHILE_()
        newWhile.expression = match.group(1)

        # Add to the parse tree
        curnode = self._top_tpl_()
        curnode.blk = newWhile

        # Push onto the stack
        self._push_node_(newWhile)
        self._push_node_(newWhile.tpl)

    def _on_endwhile_(self, contents):
        if contents:
            raise EInvalidEmptyTag(contents)

        self._finish_block_( 'while' )
        
    def _on_def_(self, contents):
        match = self.def_re.search(contents)
        if not match:
            raise EInvalidDefTag(contents)

        defname = match.group(1)

        # Get the args
        # Strip the unnecessary whitespace from the args
        args = string.split(match.group(2), ',')
        args = map(lambda arg: string.strip(arg), args)

        if self.fileobj.root().def_exists( defname ):
            raise EDefAlreadyExists( defname )

        # Create a new def object:
        newDef = _DEF_()
        newDef.funcname = defname
        newDef.args = args

        # Add to the parse tree:
        curtpl = self._top_tpl_()
        curtpl.blk = newDef

        # Push onto the stack:
        self._push_node_(newDef)
        self._push_node_(newDef.tpl)

    def _on_enddef_(self, contents):
        if contents:
            raise EInvalidEmptyTag(contents)

        def_tree = self._finish_block_( 'def', detach=1 )

        # Add this to the def list.  We have the root of the tree, and
        # we told _finish_block_ to detach it from the main tree, so
        # we can safely pass it off to the def list in the root file.
        self.fileobj.root().def_add( def_tree.funcname, def_tree )

    def _on_echo_(self, contents):
        # Simply add an echo tag to the parse tree.  Add a tpl to the current
        # tpl and push that.
        newEcho = _ECHO_()
        newEcho.expression = contents
        self._insert_atomic_tag_( newEcho )

    def _on_comment_(self, contents):
        # Throw it away:
        # XXX: We may want to keep this for use later on, but right now we
        # just throw it away and never see it again.  The tokenizer doesn't
        # ignore it, but the parser does.  It never gets into the tree.
        pass

    def _on_call_(self, contents):
        match = self.call_re.search(contents)
        if not match:
            raise EInvalidCallTag(contents)

        # Create and add a new call block
        newCall = _CALL_()
        newCall.funcname = match.group(1)
        newCall.args = string.split(match.group(2), ',')

        self._insert_atomic_tag_(newCall)

    def _on_inc_(self, contents):
        match = self.inc_re.search(contents)
        if not match:
            raise EInvalidIncTag(contents)

        filename = self._unquote_inc_(contents)
        abs_name = self.fileobj.find_file( filename )

        if self.fileobj.is_file_included( abs_name ):
            raise ERecursiveInclusion( abs_name, self.fileobj.filename() )

        rootfile = self.fileobj.root()
        parse_tree = None
        if rootfile.include_exists( abs_name ):
            rootfile.include_get( abs_name )['tree']
        else:
            # Not found?  We need to parse the sucker
            # Note that a reference to the fileobj doesn't introduce
            # circular references because this is tied to a local
            # variable that will go out of scope almost immediately.
            tfile = _TemplateFile_(self.fileobj)
            tfile.parse( abs_name )
            parse_tree = tfile.parse_tree()

            # Now add it to the root's include table:
            rootfile.include_add(
                tfile.filename(),
                tfile.filetime(),
                tfile.parse_tree()
                )
        
        # Add the filename and a reference to the include to the parse tree
        newInc = _INC_()
        newInc.filename = abs_name
        newInc.parse_tree = tfile.parse_tree()

        self._insert_atomic_tag_(newInc)

    def _on_rawinc_(self, contents):
        match = self.inc_re.search(contents)
        if not match:
            raise EInvalidRawIncTag(contents)

        filename = self._unquote_inc_(contents)
        abs_name = self.fileobj.find_file( filename )

        # Find out when this file was last modified.  We store that.
        mtime = os.stat( abs_name )[stat.ST_MTIME]

        rootfile = self.fileobj.root()
        # We don't store the text of the file in the parse tree.  We just
        # store the filename.  The text of the file will be included in
        # the code generated for the template as a member of a global
        # dictionary.  This keeps us from using more memory than necessary.

        # We do store the filename in the dictionary.  The contents are
        # only read when the code is generated.

        if not rootfile.raw_exists( abs_name ):
            rootfile.raw_add( abs_name, mtime )
            
        newRawInc = _RAWINC_()
        newRawInc.filename = abs_name

        self._insert_atomic_tag_(newRawInc)

    def _on_exec_(self, contents):
        # We don't parse this at all.  We just dump it as is and hope that it
        # works correctly.

        newExec = _EXEC_()
        newExec.expression = contents

        self._insert_atomic_tag_(newExec)

    def _on_break_(self, contents):
        if contents:
            raise EInvalidEmptyTag(contents)

        self._insert_atomic_tag_(_BREAK_())

    def _on_continue_(self, contents):
        if contents:
            raise EInvalidEmptyTag(contents)

        self._insert_atomic_tag_(_CONTINUE_())

# -----------------------------------------------------------------------------
# -----------------------------------------------------------------------------
class Template:
    """Main Template class:
        Interact with this class.  All other classes in this module
        are support classes."""

    class __Namespace: pass

    def __init__(self, filename, path=None, keep_code=0):
        """Creates a new template instance.  This instance is NOT a template
        namespace, which allows you to output the text of a template.  This
        is instead an object which allows you to create namespaces for
        the template file that is parsed.

        The parsing happens when the instance is created.  It also attempts
        to compile the generated code, and throws that code away unless
        explicitly told not to (pass in keep_code=1)."""

        self._original_filename = filename
        self._keep_code = keep_code
        self._path = path

        # Initially, we don't know what the absolute filename will be.
        self._filename = None

        self.reload()

    def reload( self ):
        # Figure out what filename to use.
        filename = self._original_filename
        if self._filename:
            filename = self._filename
            
        # Note that we use only local variables here for the root template.
        # This is because we don't want to keep it around.  We'll parse it
        # and get some important info out of it, and then we'll discard it.
        # It keeps the template parse tree from sticking around in memory.
        # Get the file and parse it
        root = _RootTemplateFile_( path=self._path )
        root.parse( filename )

        # Try to compile it into a code object
        python_code = root.output_python()
        self._code_object = compile( python_code, '<string>', 'exec' )

        # Store information about the includes and the root file
        self._includes = root.include_dict() or {}
        self._filename = root.filename()
        self._filetime = root.filetime()

        # If we are supposed to keep the generated code around, we do that.
        # This will usually not take up too much memory.
        if self._keep_code:
            self._python_code = python_code
        else:
            self.remove_python_code()

    def new_namespace( self ):
        """Create a new namespace for this template.  This creates an object
        that can have variables set in it and that has the output functions.
        Once you have created a new namespace, you can output the template
        as many times as you like."""
        n = self.__Namespace()
        exec self._code_object in n.__dict__

        return n

    def includes( self ):
        """Returns a dictionary of included files (keys)
        and their mtimes (values)."""
        return self._includes

    def filename( self ):
        """Return the filename that belongs to this template.  It will be
        the full path of the file."""
        return self._filename

    def filetime( self ):
        """Return the mtime of this file when it was parsed."""
        return self._filetime

    def python_code( self ):
        """Returns the generated python code (text)."""
        return self._python_code

    def remove_python_code( self ):
        """Releases the memory used to hold onto the generated python code
        (the text code, not the compiled object)."""
        self._python_code = '# PYTHON CODE NOT AVAILABLE (keep_code=0)\n'

    def python_code_object( self ):
        """Returns the compiled code object."""
        return self._code_object

    def is_root_modified( self ):
        """Returns true if the root file has been modified since the last time
        it was parsed."""
        return os.stat(self.filename())[stat.ST_MTIME] > self.filetime()

    def is_include_modified( self ):
        """Returns true if any of the includes have been modified since the
        last time this template was parsed."""
        inc = self.includes()
        for i in inc.keys():
            if os.stat(i)[stat.ST_MTIME] > inc[i]:
                return 1
        return 0

    def is_modified( self ):
        return self.is_root_modified() or self.is_include_modified()
# -----------------------------------------------------------------------------
# -----------------------------------------------------------------------------
