Reimplement the old and horrible bison parser for .ll files with a nice

and clean recursive descent parser.

This change has a couple of ramifications:
1. The parser code is about 400 lines shorter (in what we maintain, not
   including what is autogenerated).
2. The code should be significantly faster than the old code because we 
   don't have to work around bison's poor handling of datatypes with 
   ctors/dtors.  This also makes the code much more resistant to memory 
   leaks.
3. We now get caret diagnostics from the .ll parser, woo.
4. The actual diagnostics emited from the parser are completely different
   so a bunch of testcases had to be updated.
5. I now disallow "%ty = type opaque %ty = type i32".  There was no good
   reason to support this, it was just an accident of the old 
   implementation.  I have no reason to think that anyone is actually using
   this.
6. The syntax for sticking a global variable has changed to make it 
   unambiguous.  I don't think anyone is depending on this since only clang
   supports this and it is not solid yet, so I'm not worried about anything
   breaking.
7. This gets rid of the last use of bison, and along with it the .cvs files.
   I'll prune this from the makefiles as a subsequent commit.

There are a few minor cleanups that can be done after this commit (suggestions
welcome!) but this passes dejagnu testing and is ready for its time in the
limelight.



git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@61558 91177308-0d34-0410-b5e6-96231b3b80d8

37 files changed, 3956 insertions, 15779 deletions

diff --git a/include/llvm/Assembly/Parser.h b/include/llvm/Assembly/Parser.h
index 904130d67c..17484aee34 100644
--- a/include/llvm/Assembly/Parser.h
+++ b/include/llvm/Assembly/Parser.h
@@ -20,6 +20,7 @@ namespace llvm {
 
 class Module;
 class ParseError;
+class raw_ostream;
 
 /// This function is the main interface to the LLVM Assembly Parser. It parses
 /// an ASCII file that (presumably) contains LLVM Assembly code. It returns a
@@ -29,7 +30,7 @@ class ParseError;
 /// @brief Parse LLVM Assembly from a file
 Module *ParseAssemblyFile(
   const std::string &Filename, ///< The name of the file to parse
-  ParseError* Error = 0        ///< If not null, an object to return errors in.
+  ParseError &Error            ///< If not null, an object to return errors in.
 );
 
 /// The function is a secondary interface to the LLVM Assembly Parser. It parses
@@ -39,9 +40,9 @@ Module *ParseAssemblyFile(
 /// run the verifier after parsing the file to check that it is okay.
 /// @brief Parse LLVM Assembly from a string
 Module *ParseAssemblyString(
-  const char * AsmString, ///< The string containing assembly
-  Module * M,             ///< A module to add the assembly too.
-  ParseError* Error = 0   ///< If not null, an object to return errors in.
+  const char *AsmString, ///< The string containing assembly
+  Module *M,             ///< A module to add the assembly too.
+  ParseError &Error      ///< If not null, an object to return errors in.
 );
 
 //===------------------------------------------------------------------------===
@@ -58,6 +59,8 @@ public:
   ParseError() : Filename("unknown"), Message("none"), LineNo(0), ColumnNo(0) {}
   ParseError(const ParseError &E);
 
+  void setFilename(const std::string &F) { Filename = F; }
+  
   // getMessage - Return the message passed in at construction time plus extra
   // information extracted from the options used to parse with...
   //
@@ -71,8 +74,12 @@ public:
     return Filename;
   }
 
-  void setError(const std::string &filename, const std::string &message,
-                 int LineNo = -1, int ColNo = -1);
+  void setError(const std::string &message, int lineNo = -1, int ColNo = -1,
+                const std::string &FileContents = "") {
+    Message = message;
+    LineNo = lineNo; ColumnNo = ColNo;
+    LineContents = FileContents;
+  }
 
   // getErrorLocation - Return the line and column number of the error in the
   // input source file.  The source filename can be derived from the
@@ -82,13 +89,16 @@ public:
   inline void getErrorLocation(int &Line, int &Column) const {
     Line = LineNo; Column = ColumnNo;
   }
+  
+  void PrintError(const char *ProgName, raw_ostream &S);
 
 private :
   std::string Filename;
   std::string Message;
   int LineNo, ColumnNo;                               // -1 if not relevant
+  std::string LineContents;
 
-  ParseError &operator=(const ParseError &E); // objects by reference
+  void operator=(const ParseError &E); // DO NOT IMPLEMENT
 };
 
 } // End llvm namespace
diff --git a/lib/AsmParser/LLLexer.cpp b/lib/AsmParser/LLLexer.cpp
index b8f497a5e9..da5b158468 100644
--- a/lib/AsmParser/LLLexer.cpp
+++ b/lib/AsmParser/LLLexer.cpp
@@ -12,16 +12,36 @@
 //===----------------------------------------------------------------------===//
 
 #include "LLLexer.h"
-#include "ParserInternals.h"
+#include "llvm/DerivedTypes.h"
+#include "llvm/Instruction.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/MathExtras.h"
-
-#include <list>
-#include "llvmAsmParser.h"
-
+#include "llvm/Support/raw_ostream.h"
+#include "llvm/Assembly/Parser.h"
 #include <cstring>
 using namespace llvm;
 
+bool LLLexer::Error(LocTy ErrorLoc, const std::string &Msg) const {
+  // Scan backward to find the start of the line.
+  const char *LineStart = ErrorLoc;
+  while (LineStart != CurBuf->getBufferStart() && 
+         LineStart[-1] != '\n' && LineStart[-1] != '\r')
+    --LineStart;
+  // Get the end of the line.
+  const char *LineEnd = ErrorLoc;
+  while (LineEnd != CurBuf->getBufferEnd() && 
+         LineEnd[0] != '\n' && LineEnd[0] != '\r')
+    ++LineEnd;
+  
+  unsigned LineNo = 1;
+  for (const char *FP = CurBuf->getBufferStart(); FP != ErrorLoc; ++FP)
+    if (*FP == '\n') ++LineNo;
+
+  std::string LineContents(LineStart, LineEnd); 
+  ErrorInfo.setError(Msg, LineNo, ErrorLoc-LineStart, LineContents);
+  return true;
+}
+
 //===----------------------------------------------------------------------===//
 // Helper functions.
 //===----------------------------------------------------------------------===//
@@ -30,21 +50,21 @@ using namespace llvm;
 // long representation... this does not have to do input error checking,
 // because we know that the input will be matched by a suitable regex...
 //
-static uint64_t atoull(const char *Buffer, const char *End) {
+uint64_t LLLexer::atoull(const char *Buffer, const char *End) {
   uint64_t Result = 0;
   for (; Buffer != End; Buffer++) {
     uint64_t OldRes = Result;
     Result *= 10;
     Result += *Buffer-'0';
     if (Result < OldRes) {  // Uh, oh, overflow detected!!!
-      GenerateError("constant bigger than 64 bits detected!");
+      Error("constant bigger than 64 bits detected!");
       return 0;
     }
   }
   return Result;
 }
 
-static uint64_t HexIntToVal(const char *Buffer, const char *End) {
+uint64_t LLLexer::HexIntToVal(const char *Buffer, const char *End) {
   uint64_t Result = 0;
   for (; Buffer != End; ++Buffer) {
     uint64_t OldRes = Result;
@@ -58,21 +78,15 @@ static uint64_t HexIntToVal(const char *Buffer, const char *End) {
       Result += C-'a'+10;
 
     if (Result < OldRes) {   // Uh, oh, overflow detected!!!
-      GenerateError("constant bigger than 64 bits detected!");
+      Error("constant bigger than 64 bits detected!");
       return 0;
     }
   }
   return Result;
 }
 
-// HexToFP - Convert the ascii string in hexadecimal format to the floating
-// point representation of it.
-//
-static double HexToFP(const char *Buffer, const char *End) {
-  return BitsToDouble(HexIntToVal(Buffer, End)); // Cast Hex constant to double
-}
-
-static void HexToIntPair(const char *Buffer, const char *End, uint64_t Pair[2]){
+void LLLexer::HexToIntPair(const char *Buffer, const char *End,
+                           uint64_t Pair[2]) {
   Pair[0] = 0;
   for (int i=0; i<16; i++, Buffer++) {
     assert(Buffer != End);
@@ -97,7 +111,7 @@ static void HexToIntPair(const char *Buffer, const char *End, uint64_t Pair[2]){
       Pair[1] += C-'a'+10;
   }
   if (Buffer != End)
-    GenerateError("constant bigger than 128 bits detected!");
+    Error("constant bigger than 128 bits detected!");
 }
 
 // UnEscapeLexed - Run through the specified buffer and change \xx codes to the
@@ -149,11 +163,8 @@ static const char *isLabelTail(const char *CurPtr) {
 // Lexer definition.
 //===----------------------------------------------------------------------===//
 
-// FIXME: REMOVE THIS.
-#define YYEOF 0
-#define YYERROR -2
-
-LLLexer::LLLexer(MemoryBuffer *StartBuf) : CurLineNo(1), CurBuf(StartBuf) {
+LLLexer::LLLexer(MemoryBuffer *StartBuf, ParseError &Err)
+  : CurBuf(StartBuf), ErrorInfo(Err), APFloatVal(0.0) {
   CurPtr = CurBuf->getBufferStart();
 }
 
@@ -174,34 +185,22 @@ int LLLexer::getNextChar() {
     // Otherwise, return end of file.
     --CurPtr;  // Another call to lex will return EOF again.
     return EOF;
-  case '\n':
-  case '\r':
-    // Handle the newline character by ignoring it and incrementing the line
-    // count.  However, be careful about 'dos style' files with \n\r in them.
-    // Only treat a \n\r or \r\n as a single line.
-    if ((*CurPtr == '\n' || (*CurPtr == '\r')) &&
-        *CurPtr != CurChar)
-      ++CurPtr;  // Eat the two char newline sequence.
-
-    ++CurLineNo;
-    return '\n';
   }
 }
 
 
-int LLLexer::LexToken() {
+lltok::Kind LLLexer::LexToken() {
   TokStart = CurPtr;
 
   int CurChar = getNextChar();
-
   switch (CurChar) {
   default:
     // Handle letters: [a-zA-Z_]
     if (isalpha(CurChar) || CurChar == '_')
       return LexIdentifier();
 
-    return CurChar;
-  case EOF: return YYEOF;
+    return lltok::Error;
+  case EOF: return lltok::Eof;
   case 0:
   case ' ':
   case '\t':
@@ -216,21 +215,21 @@ int LLLexer::LexToken() {
   case '.':
     if (const char *Ptr = isLabelTail(CurPtr)) {
       CurPtr = Ptr;
-      llvmAsmlval.StrVal = new std::string(TokStart, CurPtr-1);
-      return LABELSTR;
+      StrVal.assign(TokStart, CurPtr-1);
+      return lltok::LabelStr;
     }
     if (CurPtr[0] == '.' && CurPtr[1] == '.') {
       CurPtr += 2;
-      return DOTDOTDOT;
+      return lltok::dotdotdot;
     }
-    return '.';
+    return lltok::Error;
   case '$':
     if (const char *Ptr = isLabelTail(CurPtr)) {
       CurPtr = Ptr;
-      llvmAsmlval.StrVal = new std::string(TokStart, CurPtr-1);
-      return LABELSTR;
+      StrVal.assign(TokStart, CurPtr-1);
+      return lltok::LabelStr;
     }
-    return '$';
+    return lltok::Error;
   case ';':
     SkipLineComment();
     return LexToken();
@@ -238,6 +237,18 @@ int LLLexer::LexToken() {
   case '5': case '6': case '7': case '8': case '9':
   case '-':
     return LexDigitOrNegative();
+  case '=': return lltok::equal;
+  case '[': return lltok::lsquare;
+  case ']': return lltok::rsquare;
+  case '{': return lltok::lbrace;
+  case '}': return lltok::rbrace;
+  case '<': return lltok::less;
+  case '>': return lltok::greater;
+  case '(': return lltok::lparen;
+  case ')': return lltok::rparen;
+  case ',': return lltok::comma;
+  case '*': return lltok::star;
+  case '\\': return lltok::backslash;
   }
 }
 
@@ -249,10 +260,10 @@ void LLLexer::SkipLineComment() {
 }
 
 /// LexAt - Lex all tokens that start with an @ character:
-///   AtStringConstant @\"[^\"]*\"
-///   GlobalVarName    @[-a-zA-Z$._][-a-zA-Z$._0-9]*
-///   GlobalVarID      @[0-9]+
-int LLLexer::LexAt() {
+///   GlobalVar   @\"[^\"]*\"
+///   GlobalVar   @[-a-zA-Z$._][-a-zA-Z$._0-9]*
+///   GlobalVarID @[0-9]+
+lltok::Kind LLLexer::LexAt() {
   // Handle AtStringConstant: @\"[^\"]*\"
   if (CurPtr[0] == '"') {
     ++CurPtr;
@@ -261,13 +272,13 @@ int LLLexer::LexAt() {
       int CurChar = getNextChar();
 
       if (CurChar == EOF) {
-        GenerateError("End of file in global variable name");
-        return YYERROR;
+        Error("end of file in global variable name");
+        return lltok::Error;
       }
       if (CurChar == '"') {
-        llvmAsmlval.StrVal = new std::string(TokStart+2, CurPtr-1);
-        UnEscapeLexed(*llvmAsmlval.StrVal);
-        return ATSTRINGCONSTANT;
+        StrVal.assign(TokStart+2, CurPtr-1);
+        UnEscapeLexed(StrVal);
+        return lltok::GlobalVar;
       }
     }
   }
@@ -280,8 +291,8 @@ int LLLexer::LexAt() {
            CurPtr[0] == '.' || CurPtr[0] == '_')
       ++CurPtr;
 
-    llvmAsmlval.StrVal = new std::string(TokStart+1, CurPtr);   // Skip @
-    return GLOBALVAR;
+    StrVal.assign(TokStart+1, CurPtr);   // Skip @
+    return lltok::GlobalVar;
   }
 
   // Handle GlobalVarID: @[0-9]+
@@ -291,21 +302,21 @@ int LLLexer::LexAt() {
 
     uint64_t Val = atoull(TokStart+1, CurPtr);
     if ((unsigned)Val != Val)
-      GenerateError("Invalid value number (too large)!");
-    llvmAsmlval.UIntVal = unsigned(Val);
-    return GLOBALVAL_ID;
+      Error("invalid value number (too large)!");
+    UIntVal = unsigned(Val);
+    return lltok::GlobalID;
   }
 
-  return '@';
+  return lltok::Error;
 }
 
 
 /// LexPercent - Lex all tokens that start with a % character:
-///   PctStringConstant  %\"[^\"]*\"
-///   LocalVarName       %[-a-zA-Z$._][-a-zA-Z$._0-9]*
-///   LocalVarID         %[0-9]+
-int LLLexer::LexPercent() {
-  // Handle PctStringConstant: %\"[^\"]*\"
+///   LocalVar   ::= %\"[^\"]*\"
+///   LocalVar   ::= %[-a-zA-Z$._][-a-zA-Z$._0-9]*
+///   LocalVarID ::= %[0-9]+
+lltok::Kind LLLexer::LexPercent() {
+  // Handle LocalVarName: %\"[^\"]*\"
   if (CurPtr[0] == '"') {
     ++CurPtr;
 
@@ -313,13 +324,13 @@ int LLLexer::LexPercent() {
       int CurChar = getNextChar();
 
       if (CurChar == EOF) {
-        GenerateError("End of file in local variable name");
-        return YYERROR;
+        Error("end of file in string constant");
+        return lltok::Error;
       }
       if (CurChar == '"') {
-        llvmAsmlval.StrVal = new std::string(TokStart+2, CurPtr-1);
-        UnEscapeLexed(*llvmAsmlval.StrVal);
-        return PCTSTRINGCONSTANT;
+        StrVal.assign(TokStart+2, CurPtr-1);
+        UnEscapeLexed(StrVal);
+        return lltok::LocalVar;
       }
     }
   }
@@ -332,8 +343,8 @@ int LLLexer::LexPercent() {
            CurPtr[0] == '.' || CurPtr[0] == '_')
       ++CurPtr;
 
-    llvmAsmlval.StrVal = new std::string(TokStart+1, CurPtr);   // Skip %
-    return LOCALVAR;
+    StrVal.assign(TokStart+1, CurPtr);   // Skip %
+    return lltok::LocalVar;
   }
 
   // Handle LocalVarID: %[0-9]+
@@ -343,38 +354,38 @@ int LLLexer::LexPercent() {
 
     uint64_t Val = atoull(TokStart+1, CurPtr);
     if ((unsigned)Val != Val)
-      GenerateError("Invalid value number (too large)!");
-    llvmAsmlval.UIntVal = unsigned(Val);
-    return LOCALVAL_ID;
+      Error("invalid value number (too large)!");
+    UIntVal = unsigned(Val);
+    return lltok::LocalVarID;
   }
 
-  return '%';
+  return lltok::Error;
 }
 
 /// LexQuote - Lex all tokens that start with a " character:
 ///   QuoteLabel        "[^"]+":
 ///   StringConstant    "[^"]*"
-int LLLexer::LexQuote() {
+lltok::Kind LLLexer::LexQuote() {
   while (1) {
     int CurChar = getNextChar();
 
     if (CurChar == EOF) {
-      GenerateError("End of file in quoted string");
-      return YYERROR;
+      Error("end of file in quoted string");
+      return lltok::Error;
     }
 
     if (CurChar != '"') continue;
 
     if (CurPtr[0] != ':') {
-      llvmAsmlval.StrVal = new std::string(TokStart+1, CurPtr-1);
-      UnEscapeLexed(*llvmAsmlval.StrVal);
-      return STRINGCONSTANT;
+      StrVal.assign(TokStart+1, CurPtr-1);
+      UnEscapeLexed(StrVal);
+      return lltok::StringConstant;
     }
 
     ++CurPtr;
-    llvmAsmlval.StrVal = new std::string(TokStart+1, CurPtr-2);
-    UnEscapeLexed(*llvmAsmlval.StrVal);
-    return LABELSTR;
+    StrVal.assign(TokStart+1, CurPtr-2);
+    UnEscapeLexed(StrVal);
+    return lltok::LabelStr;
   }
 }
 
@@ -395,7 +406,7 @@ static bool JustWhitespaceNewLine(const char *&Ptr) {
 ///    IntegerType     i[0-9]+
 ///    Keyword         sdiv, float, ...
 ///    HexIntConstant  [us]0x[0-9A-Fa-f]+
-int LLLexer::LexIdentifier() {
+lltok::Kind LLLexer::LexIdentifier() {
   const char *StartChar = CurPtr;
   const char *IntEnd = CurPtr[-1] == 'i' ? 0 : StartChar;
   const char *KeywordEnd = 0;
@@ -408,8 +419,8 @@ int LLLexer::LexIdentifier() {
 
   // If we stopped due to a colon, this really is a label.
   if (*CurPtr == ':') {
-    llvmAsmlval.StrVal = new std::string(StartChar-1, CurPtr++);
-    return LABELSTR;
+    StrVal.assign(StartChar-1, CurPtr++);
+    return lltok::LabelStr;
   }
 
   // Otherwise, this wasn't a label.  If this was valid as an integer type,
@@ -420,12 +431,11 @@ int LLLexer::LexIdentifier() {
     uint64_t NumBits = atoull(StartChar, CurPtr);
     if (NumBits < IntegerType::MIN_INT_BITS ||
         NumBits > IntegerType::MAX_INT_BITS) {
-      GenerateError("Bitwidth for integer type out of range!");
-      return YYERROR;
+      Error("bitwidth for integer type out of range!");
+      return lltok::Error;
     }
-    const Type* Ty = IntegerType::get(NumBits);
-    llvmAsmlval.PrimType = Ty;
-    return INTTYPE;
+    TyVal = IntegerType::get(NumBits);
+    return lltok::Type;
   }
 
   // Otherwise, this was a letter sequence.  See which keyword this is.
@@ -433,112 +443,96 @@ int LLLexer::LexIdentifier() {
   CurPtr = KeywordEnd;
   --StartChar;
   unsigned Len = CurPtr-StartChar;
-#define KEYWORD(STR, TOK) \
-  if (Len == strlen(STR) && !memcmp(StartChar, STR, strlen(STR))) return TOK;
-
-  KEYWORD("begin",     BEGINTOK);
-  KEYWORD("end",       ENDTOK);
-  KEYWORD("true",      TRUETOK);
-  KEYWORD("false",     FALSETOK);
-  KEYWORD("declare",   DECLARE);
-  KEYWORD("define",    DEFINE);
-  KEYWORD("global",    GLOBAL);
-  KEYWORD("constant",  CONSTANT);
-
-  KEYWORD("internal",  INTERNAL);
-  KEYWORD("linkonce",  LINKONCE);
-  KEYWORD("weak",      WEAK);
-  KEYWORD("appending", APPENDING);
-  KEYWORD("dllimport", DLLIMPORT);
-  KEYWORD("dllexport", DLLEXPORT);
-  KEYWORD("common", COMMON);
-  KEYWORD("default", DEFAULT);
-  KEYWORD("hidden", HIDDEN);
-  KEYWORD("protected", PROTECTED);
-  KEYWORD("extern_weak", EXTERN_WEAK);
-  KEYWORD("external", EXTERNAL);
-  KEYWORD("thread_local", THREAD_LOCAL);
-  KEYWORD("zeroinitializer", ZEROINITIALIZER);
-  KEYWORD("undef", UNDEF);
-  KEYWORD("null", NULL_TOK);
-  KEYWORD("to", TO);
-  KEYWORD("tail", TAIL);
-  KEYWORD("target", TARGET);
-  KEYWORD("triple", TRIPLE);
-  KEYWORD("deplibs", DEPLIBS);
-  KEYWORD("datalayout", DATALAYOUT);
-  KEYWORD("volatile", VOLATILE);
-  KEYWORD("align", ALIGN);
-  KEYWORD("addrspace", ADDRSPACE);
-  KEYWORD("section", SECTION);
-  KEYWORD("alias", ALIAS);
-  KEYWORD("module", MODULE);
-  KEYWORD("asm", ASM_TOK);
-  KEYWORD("sideeffect", SIDEEFFECT);
-  KEYWORD("gc", GC);
-
-  KEYWORD("cc", CC_TOK);
-  KEYWORD("ccc", CCC_TOK);
-  KEYWORD("fastcc", FASTCC_TOK);
-  KEYWORD("coldcc", COLDCC_TOK);
-  KEYWORD("x86_stdcallcc", X86_STDCALLCC_TOK);
-  KEYWORD("x86_fastcallcc", X86_FASTCALLCC_TOK);
-
-  KEYWORD("signext", SIGNEXT);
-  KEYWORD("zeroext", ZEROEXT);
-  KEYWORD("inreg", INREG);
-  KEYWORD("sret", SRET);
-  KEYWORD("nounwind", NOUNWIND);
-  KEYWORD("noreturn", NORETURN);
-  KEYWORD("noalias", NOALIAS);
-  KEYWORD("nocapture", NOCAPTURE);
-  KEYWORD("byval", BYVAL);
-  KEYWORD("nest", NEST);
-  KEYWORD("readnone", READNONE);
-  KEYWORD("readonly", READONLY);
-
-  KEYWORD("noinline", NOINLINE);
-  KEYWORD("alwaysinline", ALWAYSINLINE);
-  KEYWORD("optsize", OPTSIZE);
-  KEYWORD("ssp", SSP);
-  KEYWORD("sspreq", SSPREQ);
-
-  KEYWORD("type", TYPE);
-  KEYWORD("opaque", OPAQUE);
-
-  KEYWORD("eq" , EQ);
-  KEYWORD("ne" , NE);
-  KEYWORD("slt", SLT);
-  KEYWORD("sgt", SGT);
-  KEYWORD("sle", SLE);
-  KEYWORD("sge", SGE);
-  KEYWORD("ult", ULT);
-  KEYWORD("ugt", UGT);
-  KEYWORD("ule", ULE);
-  KEYWORD("uge", UGE);
-  KEYWORD("oeq", OEQ);
-  KEYWORD("one", ONE);
-  KEYWORD("olt", OLT);
-  KEYWORD("ogt", OGT);
-  KEYWORD("ole", OLE);
-  KEYWORD("oge", OGE);
-  KEYWORD("ord", ORD);
-  KEYWORD("uno", UNO);
-  KEYWORD("ueq", UEQ);
-  KEYWORD("une", UNE);
+#define KEYWORD(STR) \
+  if (Len == strlen(#STR) && !memcmp(StartChar, #STR, strlen(#STR))) \
+    return lltok::kw_##STR;
+
+  KEYWORD(begin);   KEYWORD(end);
+  KEYWORD(true);    KEYWORD(false);
+  KEYWORD(declare); KEYWORD(define);
+  KEYWORD(global);  KEYWORD(constant);
+
+  KEYWORD(internal);
+  KEYWORD(linkonce);
+  KEYWORD(weak);
+  KEYWORD(appending);
+  KEYWORD(dllimport);
+  KEYWORD(dllexport);
+  KEYWORD(common);
+  KEYWORD(default);
+  KEYWORD(hidden);
+  KEYWORD(protected);
+  KEYWORD(extern_weak);
+  KEYWORD(external);
+  KEYWORD(thread_local);
+  KEYWORD(zeroinitializer);
+  KEYWORD(undef);
+  KEYWORD(null);
+  KEYWORD(to);
+  KEYWORD(tail);
+  KEYWORD(target);
+  KEYWORD(triple);
+  KEYWORD(deplibs);
+  KEYWORD(datalayout);
+  KEYWORD(volatile);
+  KEYWORD(align);
+  KEYWORD(addrspace);
+  KEYWORD(section);
+  KEYWORD(alias);
+  KEYWORD(module);
+  KEYWORD(asm);
+  KEYWORD(sideeffect);
+  KEYWORD(gc);
+
+  KEYWORD(ccc);
+  KEYWORD(fastcc);
+  KEYWORD(coldcc);
+  KEYWORD(x86_stdcallcc);
+  KEYWORD(x86_fastcallcc);
+  KEYWORD(cc);
+  KEYWORD(c);
+
+  KEYWORD(signext);
+  KEYWORD(zeroext);
+  KEYWORD(inreg);
+  KEYWORD(sret);
+  KEYWORD(nounwind);
+  KEYWORD(noreturn);
+  KEYWORD(noalias);
+  KEYWORD(nocapture);
+  KEYWORD(byval);
+  KEYWORD(nest);
+  KEYWORD(readnone);
+  KEYWORD(readonly);
+
+  KEYWORD(noinline);
+  KEYWORD(alwaysinline);
+  KEYWORD(optsize);
+  KEYWORD(ssp);
+  KEYWORD(sspreq);
+
+  KEYWORD(type);
+  KEYWORD(opaque);
+
+  KEYWORD(eq); KEYWORD(ne); KEYWORD(slt); KEYWORD(sgt); KEYWORD(sle);
+  KEYWORD(sge); KEYWORD(ult); KEYWORD(ugt); KEYWORD(ule); KEYWORD(uge);
+  KEYWORD(oeq); KEYWORD(one); KEYWORD(olt); KEYWORD(ogt); KEYWORD(ole);
+  KEYWORD(oge); KEYWORD(ord); KEYWORD(uno); KEYWORD(ueq); KEYWORD(une);
+  
+  KEYWORD(x);
 #undef KEYWORD
 
   // Keywords for types.
-#define TYPEKEYWORD(STR, LLVMTY, TOK) \
+#define TYPEKEYWORD(STR, LLVMTY) \
   if (Len == strlen(STR) && !memcmp(StartChar, STR, strlen(STR))) { \
-    llvmAsmlval.PrimType = LLVMTY; return TOK; }
-  TYPEKEYWORD("void",      Type::VoidTy,  VOID);
-  TYPEKEYWORD("float",     Type::FloatTy, FLOAT);
-  TYPEKEYWORD("double",    Type::DoubleTy, DOUBLE);
-  TYPEKEYWORD("x86_fp80",  Type::X86_FP80Ty, X86_FP80);
-  TYPEKEYWORD("fp128",     Type::FP128Ty, FP128);
-  TYPEKEYWORD("ppc_fp128", Type::PPC_FP128Ty, PPC_FP128);
-  TYPEKEYWORD("label",     Type::LabelTy, LABEL);
+    TyVal = LLVMTY; return lltok::Type; }
+  TYPEKEYWORD("void",      Type::VoidTy);
+  TYPEKEYWORD("float",     Type::FloatTy);
+  TYPEKEYWORD("double",    Type::DoubleTy);
+  TYPEKEYWORD("x86_fp80",  Type::X86_FP80Ty);
+  TYPEKEYWORD("fp128",     Type::FP128Ty);
+  TYPEKEYWORD("ppc_fp128", Type::PPC_FP128Ty);
+  TYPEKEYWORD("label",     Type::LabelTy);
 #undef TYPEKEYWORD
 
   // Handle special forms for autoupgrading.  Drop these in LLVM 3.0.  This is
@@ -546,74 +540,62 @@ int LLLexer::LexIdentifier() {
   if (Len == 4 && !memcmp(StartChar, "sext", 4)) {
     // Scan CurPtr ahead, seeing if there is just whitespace before the newline.
     if (JustWhitespaceNewLine(CurPtr))
-      return SIGNEXT;
+      return lltok::kw_signext;
   } else if (Len == 4 && !memcmp(StartChar, "zext", 4)) {
     // Scan CurPtr ahead, seeing if there is just whitespace before the newline.
     if (JustWhitespaceNewLine(CurPtr))
-      return ZEROEXT;
+      return lltok::kw_zeroext;
   }
 
   // Keywords for instructions.
-#define INSTKEYWORD(STR, type, Enum, TOK) \
-  if (Len == strlen(STR) && !memcmp(StartChar, STR, strlen(STR))) { \
-    llvmAsmlval.type = Instruction::Enum; return TOK; }
-
-  INSTKEYWORD("add",     BinaryOpVal, Add, ADD);
-  INSTKEYWORD("sub",     BinaryOpVal, Sub, SUB);
-  INSTKEYWORD("mul",     BinaryOpVal, Mul, MUL);
-  INSTKEYWORD("udiv",    BinaryOpVal, UDiv, UDIV);
-  INSTKEYWORD("sdiv",    BinaryOpVal, SDiv, SDIV);
-  INSTKEYWORD("fdiv",    BinaryOpVal, FDiv, FDIV);
-  INSTKEYWORD("urem",    BinaryOpVal, URem, UREM);
-  INSTKEYWORD("srem",    BinaryOpVal, SRem, SREM);
-  INSTKEYWORD("frem",    BinaryOpVal, FRem, FREM);
-  INSTKEYWORD("shl",     BinaryOpVal, Shl, SHL);
-  INSTKEYWORD("lshr",    BinaryOpVal, LShr, LSHR);
-  INSTKEYWORD("ashr",    BinaryOpVal, AShr, ASHR);
-  INSTKEYWORD("and",     BinaryOpVal, And, AND);
-  INSTKEYWORD("or",      BinaryOpVal, Or , OR );
-  INSTKEYWORD("xor",     BinaryOpVal, Xor, XOR);
-  INSTKEYWORD("icmp",    OtherOpVal,  ICmp,  ICMP);
-  INSTKEYWORD("fcmp",    OtherOpVal,  FCmp,  FCMP);
-  INSTKEYWORD("vicmp",   OtherOpVal,  VICmp, VICMP);
-  INSTKEYWORD("vfcmp",   OtherOpVal,  VFCmp, VFCMP);
-
-  INSTKEYWORD("phi",         OtherOpVal, PHI, PHI_TOK);
-  INSTKEYWORD("call",        OtherOpVal, Call, CALL);
-  INSTKEYWORD("trunc",       CastOpVal, Trunc, TRUNC);
-  INSTKEYWORD("zext",        CastOpVal, ZExt, ZEXT);
-  INSTKEYWORD("sext",        CastOpVal, SExt, SEXT);
-  INSTKEYWORD("fptrunc",     CastOpVal, FPTrunc, FPTRUNC);
-  INSTKEYWORD("fpext",       CastOpVal, FPExt, FPEXT);
-  INSTKEYWORD("uitofp",      CastOpVal, UIToFP, UITOFP);
-  INSTKEYWORD("sitofp",      CastOpVal, SIToFP, SITOFP);
-  INSTKEYWORD("fptoui",      CastOpVal, FPToUI, FPTOUI);
-  INSTKEYWORD("fptosi",      CastOpVal, FPToSI, FPTOSI);
-  INSTKEYWORD("inttoptr",    CastOpVal, IntToPtr, INTTOPTR);
-  INSTKEYWORD("ptrtoint",    CastOpVal, PtrToInt, PTRTOINT);
-  INSTKEYWORD("bitcast",     CastOpVal, BitCast, BITCAST);
-  INSTKEYWORD("select",      OtherOpVal, Select, SELECT);
-  INSTKEYWORD("va_arg",      OtherOpVal, VAArg , VAARG);
-  INSTKEYWORD("ret",         TermOpVal, Ret, RET);
-  INSTKEYWORD("br",          TermOpVal, Br, BR);
-  INSTKEYWORD("switch",      TermOpVal, Switch, SWITCH);
-  INSTKEYWORD("invoke",      TermOpVal, Invoke, INVOKE);
-  INSTKEYWORD("unwind",      TermOpVal, Unwind, UNWIND);
-  INSTKEYWORD("unreachable", TermOpVal, Unreachable, UNREACHABLE);
-
-  INSTKEYWORD("malloc",      MemOpVal, Malloc, MALLOC);
-  INSTKEYWORD("alloca",      MemOpVal, Alloca, ALLOCA);
-  INSTKEYWORD("free",        MemOpVal, Free, FREE);