{-# LANGUAGE CPP, DeriveDataTypeable #-}
-----------------------------------------------------------------------------
-- |
-- Module      : Language.Python.Common.Token 
-- Copyright   : (c) 2009 Bernie Pope 
-- License     : BSD-style
-- Maintainer  : bjpop@csse.unimelb.edu.au
-- Stability   : experimental
-- Portability : ghc
--
-- Lexical tokens for the Python lexer. Contains the superset of tokens from
-- version 2 and version 3 of Python (they are mostly the same).
-----------------------------------------------------------------------------

module Language.Python.Common.Token ( 
   -- * The tokens 
   Token (..),
   -- * String conversion
   debugTokenString,
   tokenString,
   -- * Classification
   hasLiteral,
   TokenClass (..),
   classifyToken
   ) where

import Language.Python.Common.Pretty
import Language.Python.Common.SrcLocation (SrcSpan (..), Span(getSpan))
import Data.Data

-- | Lexical tokens.
data Token 
   -- Whitespace
   = IndentToken { Token -> SrcSpan
token_span :: !SrcSpan }                       -- ^ Indentation: increase.
   | DedentToken { token_span :: !SrcSpan }                       -- ^ Indentation: decrease.
   | NewlineToken { token_span :: !SrcSpan }                      -- ^ Newline.
   | LineJoinToken { token_span :: !SrcSpan }                     -- ^ Line join (backslash at end of line).

   -- Comment
   | CommentToken { token_span :: !SrcSpan, Token -> String
token_literal :: !String } -- ^ Single line comment.

   -- Identifiers 
   | IdentifierToken { token_span :: !SrcSpan, token_literal :: !String }            -- ^ Identifier.

   -- Literals
   | StringToken { token_span :: !SrcSpan, token_literal :: !String }                   -- ^ Literal: string.
   | ByteStringToken { token_span :: !SrcSpan, token_literal :: !String }    -- ^ Literal: byte string.
   | UnicodeStringToken { token_span :: !SrcSpan, token_literal :: !String } -- ^ Literal: unicode string, version 2 only.
   | IntegerToken { token_span :: !SrcSpan, token_literal :: !String, Token -> Integer
token_integer :: !Integer }                 -- ^ Literal: integer.
   | LongIntegerToken { token_span :: !SrcSpan, token_literal :: !String, token_integer :: !Integer }             -- ^ Literal: long integer. /Version 2 only/.
   | FloatToken { token_span :: !SrcSpan, token_literal :: !String, Token -> Double
token_double :: !Double }                     -- ^ Literal: floating point.
   | ImaginaryToken { token_span :: !SrcSpan, token_literal :: !String, token_double :: !Double }                 -- ^ Literal: imaginary number.

   -- Keywords
   | DefToken { token_span :: !SrcSpan }                          -- ^ Keyword: \'def\'. 
   | WhileToken { token_span :: !SrcSpan }                        -- ^ Keyword: \'while\'.
   | IfToken { token_span :: !SrcSpan }                           -- ^ Keyword: \'if\'.
   | TrueToken { token_span :: !SrcSpan }                         -- ^ Keyword: \'True\'.
   | FalseToken { token_span :: !SrcSpan }                        -- ^ Keyword: \'False\'.
   | ReturnToken { token_span :: !SrcSpan }                       -- ^ Keyword: \'Return\'.
   | TryToken { token_span :: !SrcSpan }                          -- ^ Keyword: \'try\'.
   | ExceptToken { token_span :: !SrcSpan }                       -- ^ Keyword: \'except\'.
   | RaiseToken { token_span :: !SrcSpan }                        -- ^ Keyword: \'raise\'.
   | InToken { token_span :: !SrcSpan }                           -- ^ Keyword: \'in\'.
   | IsToken { token_span :: !SrcSpan }                           -- ^ Keyword: \'is\'.
   | LambdaToken { token_span :: !SrcSpan }                       -- ^ Keyword: \'lambda\'.
   | ClassToken { token_span :: !SrcSpan }                        -- ^ Keyword: \'class\'.
   | FinallyToken { token_span :: !SrcSpan }                      -- ^ Keyword: \'finally\'.
   | NoneToken { token_span :: !SrcSpan }                         -- ^ Keyword: \'None\'. 
   | ForToken { token_span :: !SrcSpan }                          -- ^ Keyword: \'for\'.
   | FromToken { token_span :: !SrcSpan }                         -- ^ Keyword: \'from\'.
   | GlobalToken { token_span :: !SrcSpan }                       -- ^ Keyword: \'global\'.
   | WithToken { token_span :: !SrcSpan }                         -- ^ Keyword: \'with\'.
   | AsToken { token_span :: !SrcSpan }                           -- ^ Keyword: \'as\'.
   | ElifToken { token_span :: !SrcSpan }                         -- ^ Keyword: \'elif\'.
   | YieldToken { token_span :: !SrcSpan }                        -- ^ Keyword: \'yield\'.
   | AssertToken { token_span :: !SrcSpan }                       -- ^ Keyword: \'assert\'.
   | ImportToken { token_span :: !SrcSpan }                       -- ^ Keyword: \'import\'.
   | PassToken { token_span :: !SrcSpan }                         -- ^ Keyword: \'pass\'.
   | BreakToken { token_span :: !SrcSpan }                        -- ^ Keyword: \'break\'.
   | ContinueToken { token_span :: !SrcSpan }                     -- ^ Keyword: \'continue\'.
   | DeleteToken { token_span :: !SrcSpan }                       -- ^ Keyword: \'del\'.
   | ElseToken { token_span :: !SrcSpan }                         -- ^ Keyword: \'else\'.
   | NotToken { token_span :: !SrcSpan }                          -- ^ Keyword: \'not\'.
   | AndToken { token_span :: !SrcSpan }                          -- ^ Keyword: boolean conjunction \'and\'.
   | OrToken { token_span :: !SrcSpan }                           -- ^ Keyword: boolean disjunction \'or\'.
   -- Version 3.x only:
   | NonLocalToken { token_span :: !SrcSpan }                     -- ^ Keyword: \'nonlocal\' (Python 3.x only)
   | AsyncToken { token_span :: !SrcSpan }                        -- ^ Keyword: \'async\' (Python 3.x only)
   | AwaitToken { token_span :: !SrcSpan }                        -- ^ Keyword: \'await\' (Python 3.x only)
   -- Version 2.x only:
   | PrintToken { token_span :: !SrcSpan }                        -- ^ Keyword: \'print\'. (Python 2.x only)
   | ExecToken { token_span :: !SrcSpan }                         -- ^ Keyword: \'exec\'. (Python 2.x only)

   -- Delimiters
   | AtToken { token_span :: !SrcSpan }                           -- ^ Delimiter: at sign \'\@\'. 
   | LeftRoundBracketToken { token_span :: !SrcSpan }             -- ^ Delimiter: left round bracket \'(\'.
   | RightRoundBracketToken { token_span :: !SrcSpan }            -- ^ Delimiter: right round bracket \')\'.
   | LeftSquareBracketToken { token_span :: !SrcSpan }            -- ^ Delimiter: left square bracket \'[\'.
   | RightSquareBracketToken { token_span :: !SrcSpan }           -- ^ Delimiter: right square bracket \']\'.
   | LeftBraceToken { token_span :: !SrcSpan }                    -- ^ Delimiter: left curly bracket \'{\'.
   | RightBraceToken { token_span :: !SrcSpan }                   -- ^ Delimiter: right curly bracket \'}\'.
   | DotToken { token_span :: !SrcSpan }                          -- ^ Delimiter: dot (full stop) \'.\'.
   | CommaToken { token_span :: !SrcSpan }                        -- ^ Delimiter: comma \',\'.
   | SemiColonToken { token_span :: !SrcSpan }                    -- ^ Delimiter: semicolon \';\'.
   | ColonToken { token_span :: !SrcSpan }                        -- ^ Delimiter: colon \':\'.
   | EllipsisToken { token_span :: !SrcSpan }                     -- ^ Delimiter: ellipses (three dots) \'...\'.
   | RightArrowToken { token_span :: !SrcSpan }                   -- ^ Delimiter: right facing arrow \'->\'.
   | AssignToken { token_span :: !SrcSpan }                       -- ^ Delimiter: assignment \'=\'.
   | PlusAssignToken { token_span :: !SrcSpan }                   -- ^ Delimiter: plus assignment \'+=\'.
   | MinusAssignToken { token_span :: !SrcSpan }                  -- ^ Delimiter: minus assignment \'-=\'.
   | MultAssignToken { token_span :: !SrcSpan }                   -- ^ Delimiter: multiply assignment \'*=\'
   | DivAssignToken { token_span :: !SrcSpan }                    -- ^ Delimiter: divide assignment \'/=\'.
   | ModAssignToken { token_span :: !SrcSpan }                    -- ^ Delimiter: modulus assignment \'%=\'.
   | PowAssignToken { token_span :: !SrcSpan }                    -- ^ Delimiter: power assignment \'**=\'.
   | BinAndAssignToken { token_span :: !SrcSpan }                 -- ^ Delimiter: binary-and assignment \'&=\'.
   | BinOrAssignToken { token_span :: !SrcSpan }                  -- ^ Delimiter: binary-or assignment \'|=\'.
   | BinXorAssignToken { token_span :: !SrcSpan }                 -- ^ Delimiter: binary-xor assignment \'^=\'.
   | LeftShiftAssignToken { token_span :: !SrcSpan }              -- ^ Delimiter: binary-left-shift assignment \'<<=\'.
   | RightShiftAssignToken { token_span :: !SrcSpan }             -- ^ Delimiter: binary-right-shift assignment \'>>=\'.
   | FloorDivAssignToken { token_span :: !SrcSpan }               -- ^ Delimiter: floor-divide assignment \'//=\'.
   | MatrixMultAssignToken { token_span :: !SrcSpan }             -- ^ Delimiter: matrix multiplication assignment \'@=\'.
   | BackQuoteToken { token_span :: !SrcSpan }                    -- ^ Delimiter: back quote character \'`\'.

   -- Operators
   | PlusToken { token_span :: !SrcSpan }                         -- ^ Operator: plus \'+\'.
   | MinusToken { token_span :: !SrcSpan }                        -- ^ Operator: minus: \'-\'.
   | MultToken { token_span :: !SrcSpan }                         -- ^ Operator: multiply \'*\'.
   | DivToken { token_span :: !SrcSpan }                          -- ^ Operator: divide \'/\'.
   | GreaterThanToken { token_span :: !SrcSpan }                  -- ^ Operator: greater-than \'>\'.
   | LessThanToken { token_span :: !SrcSpan }                     -- ^ Operator: less-than \'<\'.
   | EqualityToken { token_span :: !SrcSpan }                     -- ^ Operator: equals \'==\'.
   | GreaterThanEqualsToken { token_span :: !SrcSpan }            -- ^ Operator: greater-than-or-equals \'>=\'.
   | LessThanEqualsToken { token_span :: !SrcSpan }               -- ^ Operator: less-than-or-equals \'<=\'.
   | ExponentToken { token_span :: !SrcSpan }                     -- ^ Operator: exponential \'**\'.
   | BinaryOrToken { token_span :: !SrcSpan }                     -- ^ Operator: binary-or \'|\'.
   | XorToken { token_span :: !SrcSpan }                          -- ^ Operator: binary-xor \'^\'.
   | BinaryAndToken { token_span :: !SrcSpan }                    -- ^ Operator: binary-and \'&\'.
   | ShiftLeftToken { token_span :: !SrcSpan }                    -- ^ Operator: binary-shift-left \'<<\'.
   | ShiftRightToken { token_span :: !SrcSpan }                   -- ^ Operator: binary-shift-right \'>>\'.
   | ModuloToken { token_span :: !SrcSpan }                       -- ^ Operator: modulus \'%\'.
   | FloorDivToken { token_span :: !SrcSpan }                     -- ^ Operator: floor-divide \'//\'.
   | TildeToken { token_span :: !SrcSpan }                        -- ^ Operator: tilde \'~\'.
   | NotEqualsToken { token_span :: !SrcSpan }                    -- ^ Operator: not-equals \'!=\'.
   | NotEqualsV2Token { token_span :: !SrcSpan }                  -- ^ Operator: not-equals \'<>\'. Version 2 only.

   -- Special cases
   | EOFToken { token_span :: !SrcSpan }                          -- ^ End of file 
   deriving (Token -> Token -> Bool
(Token -> Token -> Bool) -> (Token -> Token -> Bool) -> Eq Token
forall a. (a -> a -> Bool) -> (a -> a -> Bool) -> Eq a
/= :: Token -> Token -> Bool
$c/= :: Token -> Token -> Bool
== :: Token -> Token -> Bool
$c== :: Token -> Token -> Bool
Eq,Eq Token
Eq Token =>
(Token -> Token -> Ordering)
-> (Token -> Token -> Bool)
-> (Token -> Token -> Bool)
-> (Token -> Token -> Bool)
-> (Token -> Token -> Bool)
-> (Token -> Token -> Token)
-> (Token -> Token -> Token)
-> Ord Token
Token -> Token -> Bool
Token -> Token -> Ordering
Token -> Token -> Token
forall a.
Eq a =>
(a -> a -> Ordering)
-> (a -> a -> Bool)
-> (a -> a -> Bool)
-> (a -> a -> Bool)
-> (a -> a -> Bool)
-> (a -> a -> a)
-> (a -> a -> a)
-> Ord a
min :: Token -> Token -> Token
$cmin :: Token -> Token -> Token
max :: Token -> Token -> Token
$cmax :: Token -> Token -> Token
>= :: Token -> Token -> Bool
$c>= :: Token -> Token -> Bool
> :: Token -> Token -> Bool
$c> :: Token -> Token -> Bool
<= :: Token -> Token -> Bool
$c<= :: Token -> Token -> Bool
< :: Token -> Token -> Bool
$c< :: Token -> Token -> Bool
compare :: Token -> Token -> Ordering
$ccompare :: Token -> Token -> Ordering
$cp1Ord :: Eq Token
Ord,Int -> Token -> ShowS
[Token] -> ShowS
Token -> String
(Int -> Token -> ShowS)
-> (Token -> String) -> ([Token] -> ShowS) -> Show Token
forall a.
(Int -> a -> ShowS) -> (a -> String) -> ([a] -> ShowS) -> Show a
showList :: [Token] -> ShowS
$cshowList :: [Token] -> ShowS
show :: Token -> String
$cshow :: Token -> String
showsPrec :: Int -> Token -> ShowS
$cshowsPrec :: Int -> Token -> ShowS
Show,Typeable,Typeable Token
Constr
DataType
Typeable Token =>
(forall (c :: * -> *).
 (forall d b. Data d => c (d -> b) -> d -> c b)
 -> (forall g. g -> c g) -> Token -> c Token)
-> (forall (c :: * -> *).
    (forall b r. Data b => c (b -> r) -> c r)
    -> (forall r. r -> c r) -> Constr -> c Token)
-> (Token -> Constr)
-> (Token -> DataType)
-> (forall (t :: * -> *) (c :: * -> *).
    Typeable t =>
    (forall d. Data d => c (t d)) -> Maybe (c Token))
-> (forall (t :: * -> * -> *) (c :: * -> *).
    Typeable t =>
    (forall d e. (Data d, Data e) => c (t d e)) -> Maybe (c Token))
-> ((forall b. Data b => b -> b) -> Token -> Token)
-> (forall r r'.
    (r -> r' -> r) -> r -> (forall d. Data d => d -> r') -> Token -> r)
-> (forall r r'.
    (r' -> r -> r) -> r -> (forall d. Data d => d -> r') -> Token -> r)
-> (forall u. (forall d. Data d => d -> u) -> Token -> [u])
-> (forall u. Int -> (forall d. Data d => d -> u) -> Token -> u)
-> (forall (m :: * -> *).
    Monad m =>
    (forall d. Data d => d -> m d) -> Token -> m Token)
-> (forall (m :: * -> *).
    MonadPlus m =>
    (forall d. Data d => d -> m d) -> Token -> m Token)
-> (forall (m :: * -> *).
    MonadPlus m =>
    (forall d. Data d => d -> m d) -> Token -> m Token)
-> Data Token
Token -> Constr
Token -> DataType
(forall b. Data b => b -> b) -> Token -> Token
(forall d b. Data d => c (d -> b) -> d -> c b)
-> (forall g. g -> c g) -> Token -> c Token
(forall b r. Data b => c (b -> r) -> c r)
-> (forall r. r -> c r) -> Constr -> c Token
forall a.
Typeable a =>
(forall (c :: * -> *).
 (forall d b. Data d => c (d -> b) -> d -> c b)
 -> (forall g. g -> c g) -> a -> c a)
-> (forall (c :: * -> *).
    (forall b r. Data b => c (b -> r) -> c r)
    -> (forall r. r -> c r) -> Constr -> c a)
-> (a -> Constr)
-> (a -> DataType)
-> (forall (t :: * -> *) (c :: * -> *).
    Typeable t =>
    (forall d. Data d => c (t d)) -> Maybe (c a))
-> (forall (t :: * -> * -> *) (c :: * -> *).
    Typeable t =>
    (forall d e. (Data d, Data e) => c (t d e)) -> Maybe (c a))
-> ((forall b. Data b => b -> b) -> a -> a)
-> (forall r r'.
    (r -> r' -> r) -> r -> (forall d. Data d => d -> r') -> a -> r)
-> (forall r r'.
    (r' -> r -> r) -> r -> (forall d. Data d => d -> r') -> a -> r)
-> (forall u. (forall d. Data d => d -> u) -> a -> [u])
-> (forall u. Int -> (forall d. Data d => d -> u) -> a -> u)
-> (forall (m :: * -> *).
    Monad m =>
    (forall d. Data d => d -> m d) -> a -> m a)
-> (forall (m :: * -> *).
    MonadPlus m =>
    (forall d. Data d => d -> m d) -> a -> m a)
-> (forall (m :: * -> *).
    MonadPlus m =>
    (forall d. Data d => d -> m d) -> a -> m a)
-> Data a
forall u. Int -> (forall d. Data d => d -> u) -> Token -> u
forall u. (forall d. Data d => d -> u) -> Token -> [u]
forall r r'.
(r -> r' -> r) -> r -> (forall d. Data d => d -> r') -> Token -> r
forall r r'.
(r' -> r -> r) -> r -> (forall d. Data d => d -> r') -> Token -> r
forall (m :: * -> *).
Monad m =>
(forall d. Data d => d -> m d) -> Token -> m Token
forall (m :: * -> *).
MonadPlus m =>
(forall d. Data d => d -> m d) -> Token -> m Token
forall (c :: * -> *).
(forall b r. Data b => c (b -> r) -> c r)
-> (forall r. r -> c r) -> Constr -> c Token
forall (c :: * -> *).
(forall d b. Data d => c (d -> b) -> d -> c b)
-> (forall g. g -> c g) -> Token -> c Token
forall (t :: * -> *) (c :: * -> *).
Typeable t =>
(forall d. Data d => c (t d)) -> Maybe (c Token)
forall (t :: * -> * -> *) (c :: * -> *).
Typeable t =>
(forall d e. (Data d, Data e) => c (t d e)) -> Maybe (c Token)
$cEOFToken :: Constr
$cNotEqualsV2Token :: Constr
$cNotEqualsToken :: Constr
$cTildeToken :: Constr
$cFloorDivToken :: Constr
$cModuloToken :: Constr
$cShiftRightToken :: Constr
$cShiftLeftToken :: Constr
$cBinaryAndToken :: Constr
$cXorToken :: Constr
$cBinaryOrToken :: Constr
$cExponentToken :: Constr
$cLessThanEqualsToken :: Constr
$cGreaterThanEqualsToken :: Constr
$cEqualityToken :: Constr
$cLessThanToken :: Constr
$cGreaterThanToken :: Constr
$cDivToken :: Constr
$cMultToken :: Constr
$cMinusToken :: Constr
$cPlusToken :: Constr
$cBackQuoteToken :: Constr
$cMatrixMultAssignToken :: Constr
$cFloorDivAssignToken :: Constr
$cRightShiftAssignToken :: Constr
$cLeftShiftAssignToken :: Constr
$cBinXorAssignToken :: Constr
$cBinOrAssignToken :: Constr
$cBinAndAssignToken :: Constr
$cPowAssignToken :: Constr
$cModAssignToken :: Constr
$cDivAssignToken :: Constr
$cMultAssignToken :: Constr
$cMinusAssignToken :: Constr
$cPlusAssignToken :: Constr
$cAssignToken :: Constr
$cRightArrowToken :: Constr
$cEllipsisToken :: Constr
$cColonToken :: Constr
$cSemiColonToken :: Constr
$cCommaToken :: Constr
$cDotToken :: Constr
$cRightBraceToken :: Constr
$cLeftBraceToken :: Constr
$cRightSquareBracketToken :: Constr
$cLeftSquareBracketToken :: Constr
$cRightRoundBracketToken :: Constr
$cLeftRoundBracketToken :: Constr
$cAtToken :: Constr
$cExecToken :: Constr
$cPrintToken :: Constr
$cAwaitToken :: Constr
$cAsyncToken :: Constr
$cNonLocalToken :: Constr
$cOrToken :: Constr
$cAndToken :: Constr
$cNotToken :: Constr
$cElseToken :: Constr
$cDeleteToken :: Constr
$cContinueToken :: Constr
$cBreakToken :: Constr
$cPassToken :: Constr
$cImportToken :: Constr
$cAssertToken :: Constr
$cYieldToken :: Constr
$cElifToken :: Constr
$cAsToken :: Constr
$cWithToken :: Constr
$cGlobalToken :: Constr
$cFromToken :: Constr
$cForToken :: Constr
$cNoneToken :: Constr
$cFinallyToken :: Constr
$cClassToken :: Constr
$cLambdaToken :: Constr
$cIsToken :: Constr
$cInToken :: Constr
$cRaiseToken :: Constr
$cExceptToken :: Constr
$cTryToken :: Constr
$cReturnToken :: Constr
$cFalseToken :: Constr
$cTrueToken :: Constr
$cIfToken :: Constr
$cWhileToken :: Constr
$cDefToken :: Constr
$cImaginaryToken :: Constr
$cFloatToken :: Constr
$cLongIntegerToken :: Constr
$cIntegerToken :: Constr
$cUnicodeStringToken :: Constr
$cByteStringToken :: Constr
$cStringToken :: Constr
$cIdentifierToken :: Constr
$cCommentToken :: Constr
$cLineJoinToken :: Constr
$cNewlineToken :: Constr
$cDedentToken :: Constr
$cIndentToken :: Constr
$tToken :: DataType
gmapMo :: (forall d. Data d => d -> m d) -> Token -> m Token
$cgmapMo :: forall (m :: * -> *).
MonadPlus m =>
(forall d. Data d => d -> m d) -> Token -> m Token
gmapMp :: (forall d. Data d => d -> m d) -> Token -> m Token
$cgmapMp :: forall (m :: * -> *).
MonadPlus m =>
(forall d. Data d => d -> m d) -> Token -> m Token
gmapM :: (forall d. Data d => d -> m d) -> Token -> m Token
$cgmapM :: forall (m :: * -> *).
Monad m =>
(forall d. Data d => d -> m d) -> Token -> m Token
gmapQi :: Int -> (forall d. Data d => d -> u) -> Token -> u
$cgmapQi :: forall u. Int -> (forall d. Data d => d -> u) -> Token -> u
gmapQ :: (forall d. Data d => d -> u) -> Token -> [u]
$cgmapQ :: forall u. (forall d. Data d => d -> u) -> Token -> [u]
gmapQr :: (r' -> r -> r) -> r -> (forall d. Data d => d -> r') -> Token -> r
$cgmapQr :: forall r r'.
(r' -> r -> r) -> r -> (forall d. Data d => d -> r') -> Token -> r
gmapQl :: (r -> r' -> r) -> r -> (forall d. Data d => d -> r') -> Token -> r
$cgmapQl :: forall r r'.
(r -> r' -> r) -> r -> (forall d. Data d => d -> r') -> Token -> r
gmapT :: (forall b. Data b => b -> b) -> Token -> Token
$cgmapT :: (forall b. Data b => b -> b) -> Token -> Token
dataCast2 :: (forall d e. (Data d, Data e) => c (t d e)) -> Maybe (c Token)
$cdataCast2 :: forall (t :: * -> * -> *) (c :: * -> *).
Typeable t =>
(forall d e. (Data d, Data e) => c (t d e)) -> Maybe (c Token)
dataCast1 :: (forall d. Data d => c (t d)) -> Maybe (c Token)
$cdataCast1 :: forall (t :: * -> *) (c :: * -> *).
Typeable t =>
(forall d. Data d => c (t d)) -> Maybe (c Token)
dataTypeOf :: Token -> DataType
$cdataTypeOf :: Token -> DataType
toConstr :: Token -> Constr
$ctoConstr :: Token -> Constr
gunfold :: (forall b r. Data b => c (b -> r) -> c r)
-> (forall r. r -> c r) -> Constr -> c Token
$cgunfold :: forall (c :: * -> *).
(forall b r. Data b => c (b -> r) -> c r)
-> (forall r. r -> c r) -> Constr -> c Token
gfoldl :: (forall d b. Data d => c (d -> b) -> d -> c b)
-> (forall g. g -> c g) -> Token -> c Token
$cgfoldl :: forall (c :: * -> *).
(forall d b. Data d => c (d -> b) -> d -> c b)
-> (forall g. g -> c g) -> Token -> c Token
$cp1Data :: Typeable Token
Data)

instance Span Token where
  getSpan :: Token -> SrcSpan
getSpan = Token -> SrcSpan
token_span 
   
-- | Produce a string from a token containing detailed information. Mainly intended for debugging. 
debugTokenString :: Token -> String
debugTokenString :: Token -> String
debugTokenString token :: Token
token =
   Doc -> String
render (String -> Doc
text (Constr -> String
forall a. Show a => a -> String
show (Constr -> String) -> Constr -> String
forall a b. (a -> b) -> a -> b
$ Token -> Constr
forall a. Data a => a -> Constr
toConstr Token
token) Doc -> Doc -> Doc
<+> SrcSpan -> Doc
forall a. Pretty a => a -> Doc
pretty (Token -> SrcSpan
token_span Token
token) Doc -> Doc -> Doc
<+>
          if Token -> Bool
hasLiteral Token
token then String -> Doc
text (Token -> String
token_literal Token
token) else Doc
empty)

-- | Test if a token contains its literal source text.
hasLiteral :: Token -> Bool
hasLiteral :: Token -> Bool
hasLiteral token :: Token
token =
   case Token
token of
      CommentToken {}       -> Bool
True 
      IdentifierToken {}    -> Bool
True 
      StringToken {}        -> Bool
True 
      ByteStringToken {}    -> Bool
True
      UnicodeStringToken {} -> Bool
True
      IntegerToken {}       -> Bool
True 
      LongIntegerToken {}   -> Bool
True 
      FloatToken {}         -> Bool
True 
      ImaginaryToken  {}    -> Bool
True 
      other :: Token
other                 -> Bool
False

-- | Classification of tokens
data TokenClass
   = Comment
   | Number
   | Identifier
   | Punctuation
   | Bracket
   | Layout 
   | Keyword
   | String
   | Operator
   | Assignment
   deriving (Int -> TokenClass -> ShowS
[TokenClass] -> ShowS
TokenClass -> String
(Int -> TokenClass -> ShowS)
-> (TokenClass -> String)
-> ([TokenClass] -> ShowS)
-> Show TokenClass
forall a.
(Int -> a -> ShowS) -> (a -> String) -> ([a] -> ShowS) -> Show a
showList :: [TokenClass] -> ShowS
$cshowList :: [TokenClass] -> ShowS
show :: TokenClass -> String
$cshow :: TokenClass -> String
showsPrec :: Int -> TokenClass -> ShowS
$cshowsPrec :: Int -> TokenClass -> ShowS
Show, TokenClass -> TokenClass -> Bool
(TokenClass -> TokenClass -> Bool)
-> (TokenClass -> TokenClass -> Bool) -> Eq TokenClass
forall a. (a -> a -> Bool) -> (a -> a -> Bool) -> Eq a
/= :: TokenClass -> TokenClass -> Bool
$c/= :: TokenClass -> TokenClass -> Bool
== :: TokenClass -> TokenClass -> Bool
$c== :: TokenClass -> TokenClass -> Bool
Eq, Eq TokenClass
Eq TokenClass =>
(TokenClass -> TokenClass -> Ordering)
-> (TokenClass -> TokenClass -> Bool)
-> (TokenClass -> TokenClass -> Bool)
-> (TokenClass -> TokenClass -> Bool)
-> (TokenClass -> TokenClass -> Bool)
-> (TokenClass -> TokenClass -> TokenClass)
-> (TokenClass -> TokenClass -> TokenClass)
-> Ord TokenClass
TokenClass -> TokenClass -> Bool
TokenClass -> TokenClass -> Ordering
TokenClass -> TokenClass -> TokenClass
forall a.
Eq a =>
(a -> a -> Ordering)
-> (a -> a -> Bool)
-> (a -> a -> Bool)
-> (a -> a -> Bool)
-> (a -> a -> Bool)
-> (a -> a -> a)
-> (a -> a -> a)
-> Ord a
min :: TokenClass -> TokenClass -> TokenClass
$cmin :: TokenClass -> TokenClass -> TokenClass
max :: TokenClass -> TokenClass -> TokenClass
$cmax :: TokenClass -> TokenClass -> TokenClass
>= :: TokenClass -> TokenClass -> Bool
$c>= :: TokenClass -> TokenClass -> Bool
> :: TokenClass -> TokenClass -> Bool
$c> :: TokenClass -> TokenClass -> Bool
<= :: TokenClass -> TokenClass -> Bool
$c<= :: TokenClass -> TokenClass -> Bool
< :: TokenClass -> TokenClass -> Bool
$c< :: TokenClass -> TokenClass -> Bool
compare :: TokenClass -> TokenClass -> Ordering
$ccompare :: TokenClass -> TokenClass -> Ordering
$cp1Ord :: Eq TokenClass
Ord)

classifyToken :: Token -> TokenClass 
classifyToken :: Token -> TokenClass
classifyToken token :: Token
token = 
   case Token
token of
      IndentToken {} -> TokenClass
Layout 
      DedentToken {} -> TokenClass
Layout 
      NewlineToken {} -> TokenClass
Layout 
      CommentToken {} -> TokenClass
Comment 
      IdentifierToken {} -> TokenClass
Identifier 
      StringToken {} -> TokenClass
String 
      ByteStringToken {} -> TokenClass
String
      UnicodeStringToken {} -> TokenClass
String
      IntegerToken {} -> TokenClass
Number 
      LongIntegerToken {} -> TokenClass
Number 
      FloatToken {} -> TokenClass
Number 
      ImaginaryToken {} -> TokenClass
Number 
      DefToken {} -> TokenClass
Keyword 
      WhileToken {} -> TokenClass
Keyword 
      IfToken {} ->  TokenClass
Keyword
      TrueToken {} -> TokenClass
Keyword 
      FalseToken {} -> TokenClass
Keyword 
      ReturnToken {} -> TokenClass
Keyword 
      TryToken {} -> TokenClass
Keyword 
      ExceptToken {} -> TokenClass
Keyword 
      RaiseToken {} -> TokenClass
Keyword 
      InToken {} -> TokenClass
Keyword 
      IsToken {} -> TokenClass
Keyword 
      LambdaToken {} -> TokenClass
Keyword 
      ClassToken {} -> TokenClass
Keyword 
      FinallyToken {} -> TokenClass
Keyword 
      NoneToken {} -> TokenClass
Keyword 
      ForToken {} -> TokenClass
Keyword 
      FromToken {} -> TokenClass
Keyword 
      GlobalToken {} -> TokenClass
Keyword 
      WithToken {} -> TokenClass
Keyword 
      AsToken {} -> TokenClass
Keyword 
      ElifToken {} -> TokenClass
Keyword 
      YieldToken {} -> TokenClass
Keyword 
      AsyncToken {} -> TokenClass
Keyword
      AwaitToken {} -> TokenClass
Keyword
      AssertToken {} -> TokenClass
Keyword 
      ImportToken {} -> TokenClass
Keyword 
      PassToken {} -> TokenClass
Keyword 
      BreakToken {} -> TokenClass
Keyword 
      ContinueToken {} -> TokenClass
Keyword 
      DeleteToken {} -> TokenClass
Keyword
      ElseToken {} -> TokenClass
Keyword 
      NotToken {} -> TokenClass
Keyword 
      AndToken {} -> TokenClass
Keyword 
      OrToken {} -> TokenClass
Keyword 
      NonLocalToken {} -> TokenClass
Keyword 
      PrintToken {} -> TokenClass
Keyword 
      ExecToken {} -> TokenClass
Keyword 
      AtToken {} -> TokenClass
Keyword 
      LeftRoundBracketToken {} -> TokenClass
Bracket 
      RightRoundBracketToken {} -> TokenClass
Bracket 
      LeftSquareBracketToken {} -> TokenClass
Bracket 
      RightSquareBracketToken {} -> TokenClass
Bracket 
      LeftBraceToken {} -> TokenClass
Bracket 
      RightBraceToken {} -> TokenClass
Bracket 
      DotToken {} -> TokenClass
Operator 
      CommaToken {} -> TokenClass
Punctuation 
      SemiColonToken {} -> TokenClass
Punctuation 
      ColonToken {} -> TokenClass
Punctuation 
      EllipsisToken {} -> TokenClass
Keyword  -- What kind of thing is an ellipsis?
      RightArrowToken {} -> TokenClass
Punctuation 
      AssignToken {} -> TokenClass
Assignment 
      PlusAssignToken {} -> TokenClass
Assignment 
      MinusAssignToken {} -> TokenClass
Assignment  
      MultAssignToken {} -> TokenClass
Assignment 
      DivAssignToken {} -> TokenClass
Assignment 
      ModAssignToken {} -> TokenClass
Assignment  
      PowAssignToken {} -> TokenClass
Assignment 
      BinAndAssignToken {} -> TokenClass
Assignment 
      BinOrAssignToken {} -> TokenClass
Assignment 
      BinXorAssignToken {} -> TokenClass
Assignment 
      LeftShiftAssignToken {} -> TokenClass
Assignment 
      RightShiftAssignToken {} -> TokenClass
Assignment 
      FloorDivAssignToken {} -> TokenClass
Assignment 
      MatrixMultAssignToken {} -> TokenClass
Assignment
      BackQuoteToken {} -> TokenClass
Punctuation 
      PlusToken {} -> TokenClass
Operator 
      MinusToken {} -> TokenClass
Operator 
      MultToken {} -> TokenClass
Operator 
      DivToken {} -> TokenClass
Operator 
      GreaterThanToken {} -> TokenClass
Operator 
      LessThanToken {} -> TokenClass
Operator 
      EqualityToken {} -> TokenClass
Operator 
      GreaterThanEqualsToken {} -> TokenClass
Operator 
      LessThanEqualsToken {} -> TokenClass
Operator 
      ExponentToken {} -> TokenClass
Operator 
      BinaryOrToken {} -> TokenClass
Operator 
      XorToken {} -> TokenClass
Operator 
      BinaryAndToken {} -> TokenClass
Operator 
      ShiftLeftToken {} -> TokenClass
Operator 
      ShiftRightToken {} -> TokenClass
Operator 
      ModuloToken {} -> TokenClass
Operator 
      FloorDivToken {} -> TokenClass
Operator 
      TildeToken {} -> TokenClass
Operator 
      NotEqualsToken {} -> TokenClass
Operator 
      NotEqualsV2Token {} -> TokenClass
Operator 
      LineJoinToken {} -> TokenClass
Layout 
      EOFToken {} -> TokenClass
Layout  -- maybe a spurious classification.

-- | Produce a string from a token which is suitable for printing as Python concrete syntax.
-- /Invisible/ tokens yield an empty string.
tokenString :: Token -> String
tokenString :: Token -> String
tokenString token :: Token
token = 
   case Token
token of
      IndentToken {} -> "" 
      DedentToken {} -> ""
      NewlineToken {} -> "" 
      CommentToken {} -> Token -> String
token_literal Token
token
      IdentifierToken {} -> Token -> String
token_literal Token
token
      StringToken {} -> Token -> String
token_literal Token
token
      ByteStringToken {} -> Token -> String
token_literal Token
token
      UnicodeStringToken {} -> Token -> String
token_literal Token
token
      IntegerToken {} -> Token -> String
token_literal Token
token
      LongIntegerToken {} -> Token -> String
token_literal Token
token
      FloatToken {} -> Token -> String
token_literal Token
token
      ImaginaryToken {} -> Token -> String
token_literal Token
token
      DefToken {} -> "def"
      WhileToken {} -> "while"
      IfToken {} -> "if"
      TrueToken {} -> "True"
      FalseToken {} -> "False"
      ReturnToken {} -> "return"
      TryToken {} -> "try"
      ExceptToken {} -> "except"
      RaiseToken {} -> "raise"
      InToken {} -> "in"
      IsToken {} -> "is"
      LambdaToken {} -> "lambda"
      ClassToken {} -> "class"
      FinallyToken {} -> "finally"
      NoneToken {} -> "None"
      ForToken {} -> "for"
      FromToken {} -> "from"
      GlobalToken {} -> "global"
      WithToken {} -> "with"
      AsToken {} -> "as"
      ElifToken {} -> "elif" 
      YieldToken {} -> "yield"
      AsyncToken {} -> "async"
      AwaitToken {} -> "await"
      AssertToken {} -> "assert" 
      ImportToken {} -> "import"
      PassToken {} -> "pass" 
      BreakToken {} -> "break" 
      ContinueToken {} -> "continue"
      DeleteToken {} -> "delete"
      ElseToken {} -> "else"
      NotToken {} -> "not"
      AndToken {} -> "and"
      OrToken {} -> "or"
      NonLocalToken {} -> "nonlocal"
      PrintToken {} -> "print"
      ExecToken {} -> "exec"
      AtToken {} -> "at"
      LeftRoundBracketToken {} -> "("
      RightRoundBracketToken {} -> ")"
      LeftSquareBracketToken {} -> "["
      RightSquareBracketToken {} -> "]"
      LeftBraceToken {} -> "{"
      RightBraceToken {} -> "}"
      DotToken {} -> "."
      CommaToken {} -> ","
      SemiColonToken {} -> ";"
      ColonToken {} -> ":"
      EllipsisToken {} -> "..."
      RightArrowToken {} -> "->"
      AssignToken {} -> "="
      PlusAssignToken {} -> "+="
      MinusAssignToken {} -> "-="
      MultAssignToken {} -> "*="
      DivAssignToken {} -> "/="
      ModAssignToken {} -> "%="
      PowAssignToken {} -> "**="
      BinAndAssignToken {} -> "&="
      BinOrAssignToken {} -> "|="
      BinXorAssignToken {} -> "^="
      LeftShiftAssignToken {} -> "<<="
      RightShiftAssignToken {} -> ">>="
      FloorDivAssignToken {} -> "//=" 
      MatrixMultAssignToken {} -> "@="
      BackQuoteToken {} -> "`"
      PlusToken {} -> "+"
      MinusToken {} -> "-"
      MultToken {} -> "*"
      DivToken {} -> "/"
      GreaterThanToken {} -> ">"
      LessThanToken {} -> "<"
      EqualityToken {} -> "=="
      GreaterThanEqualsToken {} -> ">="
      LessThanEqualsToken {} -> "<="
      ExponentToken {} -> "**"
      BinaryOrToken {} -> "|"
      XorToken {} -> "^"
      BinaryAndToken {} -> "&"
      ShiftLeftToken {} -> "<<"
      ShiftRightToken {} -> ">>"
      ModuloToken {} -> "%"
      FloorDivToken {} -> "//"
      TildeToken {} -> "~"
      NotEqualsToken {} -> "!="
      NotEqualsV2Token {} -> "<>"
      LineJoinToken {} -> "\\"
      EOFToken {} -> ""