root/Cheetah/Parser.py

#!/usr/bin/env python
# $Id: Parser.py,v 1.130 2006/06/21 23:49:14 tavis_rudd Exp $
"""Parser classes for Cheetah's Compiler

Classes:
  ParseError( Exception )
  _LowLevelParser( Cheetah.SourceReader.SourceReader ), basically a lexer
  _HighLevelParser( _LowLevelParser )
  Parser === _HighLevelParser (an alias)

Meta-Data
================================================================================
Author: Tavis Rudd <tavis@damnsimple.com>
Version: $Revision: 1.130 $
Start Date: 2001/08/01
Last Revision Date: $Date: 2006/06/21 23:49:14 $
"""
__author__ = "Tavis Rudd <tavis@damnsimple.com>"
__revision__ = "$Revision: 1.130 $"[11:-2]

import os
import sys
import re
from re import DOTALL, MULTILINE
from types import StringType, ListType, TupleType, ClassType, TypeType
import time
from tokenize import pseudoprog
import inspect
import new
import traceback

from Cheetah.SourceReader import SourceReader
from Cheetah import Filters
from Cheetah import ErrorCatchers
from Cheetah.Unspecified import Unspecified

# re tools
def escapeRegexChars(txt,
                     escapeRE=re.compile(r'([\$\^\*\+\.\?\{\}\[\]\(\)\|\\])')):
    
    """Return a txt with all special regular expressions chars escaped."""
    
    return escapeRE.sub(r'\\\1' , txt)

def group(*choices): return '(' + '|'.join(choices) + ')'
def nongroup(*choices): return '(?:' + '|'.join(choices) + ')'
def namedGroup(name, *choices): return '(P:<' + name +'>' + '|'.join(choices) + ')'
def any(*choices): return apply(group, choices) + '*'
def maybe(*choices): return apply(group, choices) + '?'

##################################################
## CONSTANTS & GLOBALS ##

NO_CACHE = 0
STATIC_CACHE = 1
REFRESH_CACHE = 2

SET_LOCAL = 0
SET_GLOBAL = 1
SET_MODULE = 2

##################################################
## Tokens for the parser ##

#generic
identchars = "abcdefghijklmnopqrstuvwxyz" \
            "ABCDEFGHIJKLMNOPQRSTUVWXYZ_"
namechars = identchars + "0123456789"

#operators
powerOp = '**'
unaryArithOps = ('+', '-', '~')
binaryArithOps = ('+', '-', '/', '//','%')
shiftOps = ('>>','<<')
bitwiseOps = ('&','|','^')
assignOp = '='
augAssignOps = ('+=','-=','/=','*=', '**=','^=','%=',
          '>>=','<<=','&=','|=', )
assignmentOps = (assignOp,) + augAssignOps

compOps = ('<','>','==','!=','<=','>=', '<>', 'is', 'in',)
booleanOps = ('and','or','not')
operators = (powerOp,) + unaryArithOps + binaryArithOps \
            + shiftOps + bitwiseOps + assignmentOps \
            + compOps + booleanOps

delimeters = ('(',')','{','}','[',']',
              ',','.',':',';','=','`') + augAssignOps


keywords = ('and',       'del',       'for',       'is',        'raise',
            'assert',    'elif',      'from',      'lambda',    'return',
            'break',     'else',      'global',    'not',       'try',   
            'class',     'except',    'if',        'or',        'while',
            'continue',  'exec',      'import',    'pass',
            'def',       'finally',   'in',        'print',
            )

single3 = "'''"
double3 = '"""'

tripleQuotedStringStarts =  ("'''", '"""', 
                             "r'''", 'r"""', "R'''", 'R"""',
                             "u'''", 'u"""', "U'''", 'U"""',
                             "ur'''", 'ur"""', "Ur'''", 'Ur"""',
                             "uR'''", 'uR"""', "UR'''", 'UR"""')

tripleQuotedStringPairs = {"'''": single3, '"""': double3,
                           "r'''": single3, 'r"""': double3,
                           "u'''": single3, 'u"""': double3,
                           "ur'''": single3, 'ur"""': double3,
                           "R'''": single3, 'R"""': double3,
                           "U'''": single3, 'U"""': double3,
                           "uR'''": single3, 'uR"""': double3,
                           "Ur'''": single3, 'Ur"""': double3,
                           "UR'''": single3, 'UR"""': double3,
                           }

closurePairs= {')':'(',']':'[','}':'{'}
closurePairsRev= {'(':')','[':']','{':'}'}

##################################################
## Regex chunks for the parser ##

tripleQuotedStringREs = {}
def makeTripleQuoteRe(start, end):
    start = escapeRegexChars(start)
    end = escapeRegexChars(end)
    return re.compile(r'(?:' + start + r').*?' + r'(?:' + end + r')', re.DOTALL)

for start, end in tripleQuotedStringPairs.items():
    tripleQuotedStringREs[start] = makeTripleQuoteRe(start, end)

WS = r'[ \f\t]*'  
EOL = r'\r\n|\n|\r'
EOLZ = EOL + r'|\Z'
escCharLookBehind = nongroup(r'(?<=\A)',r'(?<!\\)')
nameCharLookAhead = r'(?=[A-Za-z_])'
identRE=re.compile(r'[a-zA-Z_][a-zA-Z_0-9]*')
EOLre=re.compile(r'(?:\r\n|\r|\n)')

specialVarRE=re.compile(r'([a-zA-z_]+)@') # for matching specialVar comments
# e.g. ##author@ Tavis Rudd

directiveNamesAndParsers = {
    # importing and inheritance
    'import':None,
    'from':None,
    'extends': 'eatExtends',
    'implements': 'eatImplements',

    # output, filtering, and caching
    'slurp': 'eatSlurp',
    'raw': 'eatRaw',
    'include': 'eatInclude',
    'cache': 'eatCache',
    'filter': 'eatFilter',
    'echo': None,
    'silent': None,
    
    'call': 'eatCall',
    'arg': 'eatCallArg',
    
    'capture': 'eatCapture',
    
    # declaration, assignment, and deletion
    'attr': 'eatAttr',
    'def': 'eatDef',
    'block': 'eatBlock',
    '@': 'eatDecorator',
    'defmacro': 'eatDefMacro',
    
    'closure': 'eatClosure',
    
    'set': 'eatSet',
    'del': None,
    
    # flow control
    'if': 'eatIf',
    'while': None,
    'for': None,
    'else': None,
    'elif': None,
    'pass': None,
    'break': None,
    'continue': None,
    'stop': None,
    'return': None,
    'yield': None,
    
    # little wrappers
    'repeat': None,
    'unless': None,
    
    # error handling
    'assert': None,
    'raise': None,
    'try': None,
    'except': None,
    'finally': None,
    'errorCatcher': 'eatErrorCatcher',
    
    # intructions to the parser and compiler
    'breakpoint': 'eatBreakPoint',
    'compiler': 'eatCompiler',
    'compiler-settings': 'eatCompilerSettings',
    
    # misc
    'shBang': 'eatShbang',
    'encoding': 'eatEncoding',
    
    'end': 'eatEndDirective',
    }

endDirectiveNamesAndHandlers = {
    'def': 'handleEndDef',      # has short-form
    'block': None,              # has short-form
    'closure': None,            # has short-form
    'cache': None,              # has short-form
    'call': None,               # has short-form
    'capture': None,            # has short-form
    'filter': None,
    'errorCatcher':None,            
    'while': None,              # has short-form
    'for': None,                # has short-form
    'if': None,                 # has short-form
    'try': None,                # has short-form
    'repeat': None,             # has short-form
    'unless': None,             # has short-form
    }

##################################################
## CLASSES ##

# @@TR: SyntaxError doesn't call exception.__str__ for some reason!
#class ParseError(SyntaxError):
class ParseError(ValueError):
    def __init__(self, stream, msg='Invalid Syntax', extMsg='', lineno=None, col=None):
        self.stream = stream
        if stream.pos() >= len(stream):
            stream.setPos(len(stream) -1)
        self.msg = msg
        self.extMsg = extMsg
        self.lineno = lineno
        self.col = col
        
    def __str__(self):
        return self.report()

    def report(self):
        stream = self.stream
        if stream.filename():
            f = " in file %s" % stream.filename()
        else:
            f = ''
        report = ''
        if self.lineno:
            lineno = self.lineno
            row, col, line = (lineno, (self.col or 0),
                              self.stream.splitlines()[lineno-1])
        else:
            row, col, line = self.stream.getRowColLine()

        ## get the surrounding lines
        lines = stream.splitlines()
        prevLines = []                  # (rowNum, content)
        for i in range(1,4):
            if row-1-i <=0:
                break
            prevLines.append( (row-i,lines[row-1-i]) )

        nextLines = []                  # (rowNum, content)
        for i in range(1,4):
            if not row-1+i < len(lines):
                break
            nextLines.append( (row+i,lines[row-1+i]) )
        nextLines.reverse()
        
        ## print the main message
        report += "\n\n%s\n" %self.msg
        report += "Line %i, column %i%s\n\n" % (row, col, f)
        report += 'Line|Cheetah Code\n'
        report += '----|-------------------------------------------------------------\n'
        while prevLines:
            lineInfo = prevLines.pop()
            report += "%(row)-4d|%(line)s\n"% {'row':lineInfo[0], 'line':lineInfo[1]}
        report += "%(row)-4d|%(line)s\n"% {'row':row, 'line':line}
        report += ' '*5 +' '*(col-1) + "^\n"
        
        while nextLines:
            lineInfo = nextLines.pop()
            report += "%(row)-4d|%(line)s\n"% {'row':lineInfo[0], 'line':lineInfo[1]}
        ## add the extra msg
        if self.extMsg:
            report += self.extMsg + '\n'
            
        return report

class ForbiddenSyntax(ParseError): pass
class ForbiddenExpression(ForbiddenSyntax): pass
class ForbiddenDirective(ForbiddenSyntax): pass

class CheetahVariable:
    def __init__(self, nameChunks, useNameMapper=True, cacheToken=None,
                 rawSource=None):
        self.nameChunks = nameChunks
        self.useNameMapper = useNameMapper
        self.cacheToken = cacheToken
        self.rawSource = rawSource
        
class Placeholder(CheetahVariable): pass

class ArgList:
    """Used by _LowLevelParser.getArgList()"""

    def __init__(self):
        self.argNames = []
        self.defVals = []
        self.i = 0

    def addArgName(self, name):
        self.argNames.append( name )
        self.defVals.append( None )

    def next(self):
        self.i += 1

    def addToDefVal(self, token):
        i = self.i
        if self.defVals[i] == None:
            self.defVals[i] = ''
        self.defVals[i] += token
    
    def merge(self):
        defVals = self.defVals
        for i in range(len(defVals)):
            if type(defVals[i]) == StringType:
                defVals[i] = defVals[i].strip()
                
        return map(None, [i.strip() for i in self.argNames], defVals)
    
    def __str__(self):
        return str(self.merge())
    
class _LowLevelParser(SourceReader):
    """This class implements the methods to match or extract ('get*') the basic
    elements of Cheetah's grammar.  It does NOT handle any code generation or
    state management.
    """

    _settingsManager = None

    def setSettingsManager(self, settingsManager):
        self._settingsManager = settingsManager
        
    def setting(self, key, default=Unspecified):
        if default is Unspecified:
            return self._settingsManager.setting(key)
        else:
            return self._settingsManager.setting(key, default=default)
        
    def setSetting(self, key, val):
        self._settingsManager.setSetting(key, val)

    def settings(self):
        return self._settingsManager.settings()
        
    def updateSettings(self, settings):
        self._settingsManager.updateSettings(settings)

    def _initializeSettings(self): 
        self._settingsManager._initializeSettings()
    
    def configureParser(self):
        """Is called by the Compiler instance after the parser has had a
        settingsManager assigned with self.setSettingsManager() 
        """
        self._makeCheetahVarREs()
        self._makeCommentREs()
        self._makeDirectiveREs()
        self._makePspREs()
        self._possibleNonStrConstantChars = (
            self.setting('commentStartToken')[0] +
            self.setting('multiLineCommentStartToken')[0] + 
            self.setting('cheetahVarStartToken')[0] +
            self.setting('directiveStartToken')[0] +
            self.setting('PSPStartToken')[0])
        self._nonStrConstMatchers = [
            self.matchCommentStartToken,
            self.matchMultiLineCommentStartToken,
            self.matchVariablePlaceholderStart,
            self.matchExpressionPlaceholderStart,
            self.matchDirective,
            self.matchPSPStartToken,
            self.matchEOLSlurpToken,
            ]

    ## regex setup ##

    def _makeCheetahVarREs(self):
        
        """Setup the regexs for Cheetah $var parsing."""

        num = r'[0-9\.]+'
        interval =   (r'(?P<interval>' + 
                      num + r's|' +
                      num + r'm|' +
                      num + r'h|' +
                      num + r'd|' +
                      num + r'w|' +
                      num + ')' 
                      )
    
        cacheToken = (r'(?:' +
                      r'(?P<REFRESH_CACHE>\*' + interval + '\*)'+
                      '|' +
                      r'(?P<STATIC_CACHE>\*)' +
                      '|' +                      
                      r'(?P<NO_CACHE>)' +
                      ')')
        self.cacheTokenRE = re.compile(cacheToken)

        silentPlaceholderToken = (r'(?:' +
                                  r'(?P<SILENT>' +escapeRegexChars('!')+')'+
                                  '|' +
                                  r'(?P<NOT_SILENT>)' +
                                  ')')
        self.silentPlaceholderTokenRE = re.compile(silentPlaceholderToken)
        
        self.cheetahVarStartRE = re.compile(
            escCharLookBehind +
            r'(?P<startToken>'+escapeRegexChars(self.setting('cheetahVarStartToken'))+')'+
            r'(?P<silenceToken>'+silentPlaceholderToken+')'+
            r'(?P<cacheToken>'+cacheToken+')'+
            r'(?P<enclosure>|(?:(?:\{|\(|\[)[ \t\f]*))' + # allow WS after enclosure
            r'(?=[A-Za-z_])')
        validCharsLookAhead = r'(?=[A-Za-z_\*!\{\(\[])'
        self.cheetahVarStartToken = self.setting('cheetahVarStartToken')
        self.cheetahVarStartTokenRE = re.compile(
            escCharLookBehind +
            escapeRegexChars(self.setting('cheetahVarStartToken'))
            +validCharsLookAhead
            )

        self.cheetahVarInExpressionStartTokenRE = re.compile(
            escapeRegexChars(self.setting('cheetahVarStartToken'))
            +r'(?=[A-Za-z_])'
            )

        self.expressionPlaceholderStartRE = re.compile(
            escCharLookBehind +
            r'(?P<startToken>' + escapeRegexChars(self.setting('cheetahVarStartToken')) + ')' +
            r'(?P<cacheToken>' + cacheToken + ')' +
            #r'\[[ \t\f]*'
            r'(?:\{|\(|\[)[ \t\f]*'
            + r'(?=[^\)\}\]])'
            )

        if self.setting('EOLSlurpToken'):
            self.EOLSlurpRE = re.compile(
                escapeRegexChars(self.setting('EOLSlurpToken'))
                + r'[ \t\f]*'
                + r'(?:'+EOL+')'
                )
        else:
            self.EOLSlurpRE = None


    def _makeCommentREs(self):
        """Construct the regex bits that are used in comment parsing."""
        startTokenEsc = escapeRegexChars(self.setting('commentStartToken'))
        self.commentStartTokenRE = re.compile(escCharLookBehind + startTokenEsc)
        del startTokenEsc
        
        startTokenEsc = escapeRegexChars(
            self.setting('multiLineCommentStartToken'))
        endTokenEsc = escapeRegexChars(
            self.setting('multiLineCommentEndToken'))
        self.multiLineCommentTokenStartRE = re.compile(escCharLookBehind +
                                                       startTokenEsc)
        self.multiLineCommentEndTokenRE = re.compile(escCharLookBehind +
                                                     endTokenEsc)
        
    def _makeDirectiveREs(self):
        """Construct the regexs that are used in directive parsing."""
        startToken = self.setting('directiveStartToken')
        endToken = self.setting('directiveEndToken')
        startTokenEsc = escapeRegexChars(startToken)
        endTokenEsc = escapeRegexChars(endToken)
        validSecondCharsLookAhead = r'(?=[A-Za-z_@])'
        reParts = [escCharLookBehind, startTokenEsc]
        if self.setting('allowWhitespaceAfterDirectiveStartToken'):
            reParts.append('[ \t]*')
        reParts.append(validSecondCharsLookAhead)
        self.directiveStartTokenRE = re.compile(''.join(reParts))
        self.directiveEndTokenRE = re.compile(escCharLookBehind + endTokenEsc)

    def _makePspREs(self):
        """Setup the regexs for PSP parsing."""
        startToken = self.setting('PSPStartToken')
        startTokenEsc = escapeRegexChars(startToken)
        self.PSPStartTokenRE = re.compile(escCharLookBehind + startTokenEsc)
        endToken = self.setting('PSPEndToken')
        endTokenEsc = escapeRegexChars(endToken)
        self.PSPEndTokenRE = re.compile(escCharLookBehind + endTokenEsc)


    def isLineClearToStartToken(self, pos=None):
        return self.isLineClearToPos(pos)

    def matchTopLevelToken(self):
        """Returns the first match found from the following methods:
            self.matchCommentStartToken
            self.matchMultiLineCommentStartToken
            self.matchVariablePlaceholderStart
            self.matchExpressionPlaceholderStart
            self.matchDirective
            self.matchPSPStartToken
            self.matchEOLSlurpToken

        Returns None if no match.
        """
        match = None
        if self.peek() in self._possibleNonStrConstantChars:
            for matcher in self._nonStrConstMatchers:
                match = matcher()
                if match:
                    break
        return match

    def matchPyToken(self):
        match = pseudoprog.match(self.src(), self.pos())
        
        if match and match.group() in tripleQuotedStringStarts:
            TQSmatch = tripleQuotedStringREs[match.group()].match(self.src(), self.pos())
            if TQSmatch:
                return TQSmatch
        return match
        
    def getPyToken(self):
        match = self.matchPyToken()
        if match is None:
            raise ParseError(self)
        elif match.group() in tripleQuotedStringStarts:
            raise ParseError(self, msg='Malformed triple-quoted string')
        return self.readTo(match.end())

    def matchEOLSlurpToken(self):
        if self.EOLSlurpRE:
            return self.EOLSlurpRE.match(self.src(), self.pos())

    def getEOLSlurpToken(self):
        match = self.matchEOLSlurpToken()
        if not match:
            raise ParseError(self, msg='Invalid EOL slurp token')
        return self.readTo(match.end())

    def matchCommentStartToken(self):
        return self.commentStartTokenRE.match(self.src(), self.pos())
    
    def getCommentStartToken(self):
        match = self.matchCommentStartToken()
        if not match:
            raise ParseError(self, msg='Invalid single-line comment start token')
        return self.readTo(match.end())

    def matchMultiLineCommentStartToken(self):
        return self.multiLineCommentTokenStartRE.match(self.src(), self.pos())
    
    def getMultiLineCommentStartToken(self):
        match = self.matchMultiLineCommentStartToken()
        if not match:
            raise ParseError(self, msg='Invalid multi-line comment start token')
        return self.readTo(match.end())

    def matchMultiLineCommentEndToken(self):
        return self.multiLineCommentEndTokenRE.match(self.src(), self.pos())
    
    def getMultiLineCommentEndToken(self):
        match = self.matchMultiLineCommentEndToken()
        if not match:
            raise ParseError(self, msg='Invalid multi-line comment end token')
        return self.readTo(match.end())
    
    def getDottedName(self):
        srcLen = len(self)
        nameChunks = []
        
        if not self.peek() in identchars:
            raise ParseError(self)
    
        while self.pos() < srcLen:
            c = self.peek()
            if c in namechars:
                nameChunk = self.getIdentifier()
                nameChunks.append(nameChunk)
            elif c == '.':
                if self.pos()+1 <srcLen and self.peek(1) in identchars:
                    nameChunks.append(self.getc())
                else:
                    break
            else:
                break

        return ''.join(nameChunks)

    def matchIdentifier(self):
        return identRE.match(self.src(), self.pos())
    
    def getIdentifier(self):
        match = self.matchIdentifier()
        if not match:
            raise ParseError(self, msg='Invalid identifier')
        return self.readTo(match.end())

    def matchOperator(self):
        match = self.matchPyToken()
        if match and match.group() not in operators:
            match = None
        return match

    def getOperator(self):
        match = self.matchOperator()
        if not match:
            raise ParseError(self, msg='Expected operator')
        return self.readTo( match.end() )

    def matchAssignmentOperator(self):
        match = self.matchPyToken()
        if match and match.group() not in assignmentOps:
            match = None
        return match
        
    def getAssignmentOperator(self):
        match = self.matchAssignmentOperator()
        if not match:
            raise ParseError(self, msg='Expected assignment operator')
        return self.readTo( match.end() )

    def matchDirective(self):
        """Returns False or the name of the directive matched.
        """
        startPos = self.pos()
        if not self.matchDirectiveStartToken():
            return False
        self.getDirectiveStartToken()
        directiveName = self.matchDirectiveName()
        self.setPos(startPos)
        return directiveName

    def matchDirectiveName(self, directiveNameChars=identchars+'0123456789-@'):
        startPos = self.pos()
        directives = self._directiveNamesAndParsers.keys()
        possibleMatches = [] 
        name = ''
        while not self.atEnd():
            c = self.getc()
            if not c in directiveNameChars:
                break
            name += c
            if name in directives:
                possibleMatches.append(name)

        possibleMatches.sort()
        possibleMatches.reverse() # longest match first
        
        directiveName = False
        if possibleMatches:
            directiveName = possibleMatches[0]
            
        self.setPos(startPos)
        return directiveName
        
    def matchDirectiveStartToken(self):
        return self.directiveStartTokenRE.match(self.src(), self.pos())
    
    def getDirectiveStartToken(self):
        match = self.matchDirectiveStartToken()
        if not match:
            raise ParseError(self, msg='Invalid directive start token')
        return self.readTo(match.end())

    def matchDirectiveEndToken(self):
        return self.directiveEndTokenRE.match(self.src(), self.pos())
    
    def getDirectiveEndToken(self):
        match = self.matchDirectiveEndToken()
        if not match:
            raise ParseError(self, msg='Invalid directive end token')
        return self.readTo(match.end())

        
    def matchColonForSingleLineShortFormDirective(self):
        if not self.atEnd() and self.peek()==':':
            restOfLine = self[self.pos()+1:self.findEOL()]
            restOfLine = restOfLine.strip()
            if not restOfLine:
                return False
            elif self.commentStartTokenRE.match(restOfLine):
                return False
            else: # non-whitespace, non-commment chars found
                return True
        return False        

    def matchPSPStartToken(self):
        return self.PSPStartTokenRE.match(self.src(), self.pos())

    def matchPSPEndToken(self):
        return self.PSPEndTokenRE.match(self.src(), self.pos())

    def getPSPStartToken(self):
        match = self.matchPSPStartToken()
        if not match:
            raise ParseError(self, msg='Invalid psp start token')
        return self.readTo(match.end())

    def getPSPEndToken(self):
        match = self.matchPSPEndToken()
        if not match:
            raise ParseError(self, msg='Invalid psp end token')
        return self.readTo(match.end())

    def matchCheetahVarStart(self):
        """includes the enclosure and cache token"""
        return self.cheetahVarStartRE.match(self.src(), self.pos())

    def matchCheetahVarStartToken(self):
        """includes the enclosure and cache token"""
        return self.cheetahVarStartTokenRE.match(self.src(), self.pos())

    def matchCheetahVarInExpressionStartToken(self):
        """no enclosures or cache tokens allowed"""
        return self.cheetahVarInExpressionStartTokenRE.match(self.src(), self.pos())

    def matchVariablePlaceholderStart(self):
        """includes the enclosure and cache token"""
        return self.cheetahVarStartRE.match(self.src(), self.pos())

    def matchExpressionPlaceholderStart(self):
        """includes the enclosure and cache token"""
        return self.expressionPlaceholderStartRE.match(self.src(), self.pos())        

    def getCheetahVarStartToken(self):
        """just the start token, not the enclosure or cache token"""
        match = self.matchCheetahVarStartToken()
        if not match:
            raise ParseError(self, msg='Expected Cheetah $var start token')            
        return self.readTo( match.end() )


    def getCacheToken(self):
        try:
            token = self.cacheTokenRE.match(self.src(), self.pos())
            self.setPos( token.end() )
            return token.group()
        except:
            raise ParseError(self, msg='Expected cache token')

    def getSilentPlaceholderToken(self):
        try:
            token = self.silentPlaceholderTokenRE.match(self.src(), self.pos())
            self.setPos( token.end() )
            return token.group()
        except:
            raise ParseError(self, msg='Expected silent placeholder token')



    def getTargetVarsList(self):
        varnames = []
        while not self.atEnd():
            if self.peek() in ' \t\f':
                self.getWhiteSpace()
            elif self.peek() in '\r\n':
                break
            elif self.startswith(','):
                self.advance()
            elif self.startswith('in ') or self.startswith('in\t'):
                break
            #elif self.matchCheetahVarStart():
            elif self.matchCheetahVarInExpressionStartToken():
                self.getCheetahVarStartToken()
                self.getSilentPlaceholderToken()
                self.getCacheToken()
                varnames.append( self.getDottedName() )
            elif self.matchIdentifier():
                varnames.append( self.getDottedName() )
            else:
                break
        return varnames
        
    def getCheetahVar(self, plain=False, skipStartToken=False):
        """This is called when parsing inside expressions. Cache tokens are only
        valid in placeholders so this method discards any cache tokens found.