From f97424b242530ad55ac06e608f0f0e7b9bad7e3a Mon Sep 17 00:00:00 2001 From: Robert Reif Date: Sat, 24 Sep 2011 14:51:03 -0400 Subject: [PATCH] start adding Windows ASCII TCHAR conversion support --- cli/cmdlineparser.cpp | 12 ++++-- lib/settings.cpp | 19 +++++----- lib/settings.h | 5 ++- lib/tokenize.cpp | 75 +++++++++++++++++++++++++++++++++++-- lib/tokenize.h | 6 +++ test/testsimplifytokens.cpp | 12 +++--- test/testtokenize.cpp | 61 +++++++++++++++++++++++------- 7 files changed, 153 insertions(+), 37 deletions(-) diff --git a/cli/cmdlineparser.cpp b/cli/cmdlineparser.cpp index ad43460c2..034b61046 100644 --- a/cli/cmdlineparser.cpp +++ b/cli/cmdlineparser.cpp @@ -596,8 +596,10 @@ bool CmdLineParser::ParseFromArgs(int argc, const char* const argv[]) { std::string platform(11+argv[i]); - if (platform == "win32") - _settings->platform(Settings::Win32); + if (platform == "win32A") + _settings->platform(Settings::Win32A); + else if (platform == "win32W") + _settings->platform(Settings::Win32W); else if (platform == "win64") _settings->platform(Settings::Win64); else if (platform == "unix32") @@ -739,8 +741,10 @@ void CmdLineParser::PrintHelp() " 32 bit unix variant\n" " * unix64\n" " 64 bit unix variant\n" - " * win32\n" - " 32 bit Windows\n" + " * win32A\n" + " 32 bit Windows ASCII character encoding\n" + " * win32W\n" + " 32 bit Windows UNICODE character encoding\n" " * win64\n" " 64 bit Windows\n" " -q, --quiet Only print error messages.\n" diff --git a/lib/settings.cpp b/lib/settings.cpp index 49d992411..9fbf45b1f 100644 --- a/lib/settings.cpp +++ b/lib/settings.cpp @@ -53,11 +53,11 @@ Settings::Settings() // This assumes the code you are checking is for the same architecture this is compiled on. #if defined(_WIN32) - platform(Win32); + platform(Win32A); #elif defined(_WIN64) platform(Win64); #else - platform(Host); + platform(Unspecified); #endif } @@ -140,8 +140,8 @@ bool Settings::platform(PlatformType type) { switch (type) { - case Host: // same as system this code was compile on - platformType = Host; + case Unspecified: // same as system this code was compile on + platformType = type; sizeof_bool = sizeof(bool); sizeof_short = sizeof(short); sizeof_int = sizeof(int); @@ -153,8 +153,9 @@ bool Settings::platform(PlatformType type) sizeof_size_t = sizeof(size_t); sizeof_pointer = sizeof(void *); return true; - case Win32: - platformType = Win32; + case Win32W: + case Win32A: + platformType = type; sizeof_bool = 1; // 4 in Visual C++ 4.2 sizeof_short = 2; sizeof_int = 4; @@ -167,7 +168,7 @@ bool Settings::platform(PlatformType type) sizeof_pointer = 4; return true; case Win64: - platformType = Win64; + platformType = type; sizeof_bool = 1; sizeof_short = 2; sizeof_int = 4; @@ -180,7 +181,7 @@ bool Settings::platform(PlatformType type) sizeof_pointer = 8; return true; case Unix32: - platformType = Unix32; + platformType = type; sizeof_bool = 1; sizeof_short = 2; sizeof_int = 4; @@ -193,7 +194,7 @@ bool Settings::platform(PlatformType type) sizeof_pointer = 4; return true; case Unix64: - platformType = Unix64; + platformType = type; sizeof_bool = 1; sizeof_short = 2; sizeof_int = 4; diff --git a/lib/settings.h b/lib/settings.h index 60852ff7a..c41c10160 100644 --- a/lib/settings.h +++ b/lib/settings.h @@ -205,8 +205,9 @@ public: enum PlatformType { - Host, // whatever system this code was compiled on - Win32, + Unspecified, // whatever system this code was compiled on + Win32A, + Win32W, Win64, Unix32, Unix64 diff --git a/lib/tokenize.cpp b/lib/tokenize.cpp index c881087b7..0aecd396a 100644 --- a/lib/tokenize.cpp +++ b/lib/tokenize.cpp @@ -2550,6 +2550,9 @@ bool Tokenizer::tokenize(std::istream &code, // convert Microsoft memory functions simplifyMicrosoftMemoryFunctions(); + // convert Microsoft string functions + simplifyMicrosoftStringFunctions(); + // Remove Qt signals and slots simplifyQtSignalsSlots(); @@ -6642,7 +6645,8 @@ void Tokenizer::simplifyPlatformTypes() } } - if (_settings->platformType == Settings::Win32 || + if (_settings->platformType == Settings::Win32A || + _settings->platformType == Settings::Win32W || _settings->platformType == Settings::Win64) { for (Token *tok = _tokens; tok; tok = tok->next()) @@ -6765,6 +6769,28 @@ void Tokenizer::simplifyPlatformTypes() } else if (tok->str() == "VOID") tok->str("void"); + else if (tok->str() == "TCHAR") + { + if (_settings->platformType == Settings::Win32A) + tok->str("char"); + } + else if (Token::Match(tok, "PTSTR|LPTSTR")) + { + if (_settings->platformType == Settings::Win32A) + { + tok->str("char"); + tok->insertToken("*"); + } + } + else if (Token::Match(tok, "PCTSTR|LPCTSTR")) + { + if (_settings->platformType == Settings::Win32A) + { + tok->str("const"); + tok->insertToken("*"); + tok->insertToken("char"); + } + } } } } @@ -10418,7 +10444,8 @@ void Tokenizer::simplifyBuiltinExpect() void Tokenizer::simplifyMicrosoftMFC() { // skip if not Windows - if (!(_settings->platformType == Settings::Win32 || + if (!(_settings->platformType == Settings::Win32A || + _settings->platformType == Settings::Win32W || _settings->platformType == Settings::Win64)) return; @@ -10443,7 +10470,8 @@ void Tokenizer::simplifyMicrosoftMFC() void Tokenizer::simplifyMicrosoftMemoryFunctions() { // skip if not Windows - if (!(_settings->platformType == Settings::Win32 || + if (!(_settings->platformType == Settings::Win32A || + _settings->platformType == Settings::Win32W || _settings->platformType == Settings::Win64)) return; @@ -10530,6 +10558,47 @@ void Tokenizer::simplifyMicrosoftMemoryFunctions() } } +void Tokenizer::simplifyMicrosoftStringFunctions() +{ + // skip if not Windows + if (_settings->platformType != Settings::Win32A) + return; + + for (Token *tok = _tokens; tok; tok = tok->next()) + { + if (Token::simpleMatch(tok, "_tcscpy (")) + { + tok->str("strcpy"); + } + else if (Token::simpleMatch(tok, "_tcscat (")) + { + tok->str("strcat"); + } + else if (Token::simpleMatch(tok, "_tcsncpy (")) + { + tok->str("strncpy"); + } + else if (Token::simpleMatch(tok, "_tcsncat (")) + { + tok->str("strncat"); + } + else if (Token::simpleMatch(tok, "_tcslen (")) + { + tok->str("strlen"); + } + else if (Token::simpleMatch(tok, "_tcsnlen (")) + { + tok->str("strnlen"); + } + else if (Token::Match(tok, "_T ( %str% )")) + { + tok->deleteThis(); + tok->deleteThis(); + tok->deleteNext(); + } + } +} + // Remove Borland code void Tokenizer::simplifyBorland() { diff --git a/lib/tokenize.h b/lib/tokenize.h index aada8ceaa..a85559054 100644 --- a/lib/tokenize.h +++ b/lib/tokenize.h @@ -627,6 +627,12 @@ public: */ void simplifyMicrosoftMemoryFunctions(); + /** + * Convert Microsoft string functions + * _tcscpy -> strcpy + */ + void simplifyMicrosoftStringFunctions(); + /** * Remove Borland code */ diff --git a/test/testsimplifytokens.cpp b/test/testsimplifytokens.cpp index 17c8e94fe..0241c106e 100644 --- a/test/testsimplifytokens.cpp +++ b/test/testsimplifytokens.cpp @@ -384,7 +384,7 @@ private: TEST_CASE(removeRedundantFor); } - std::string tok(const char code[], bool simplify = true, Settings::PlatformType type = Settings::Host) + std::string tok(const char code[], bool simplify = true, Settings::PlatformType type = Settings::Unspecified) { errout.str(""); @@ -863,7 +863,7 @@ private: // Simplify 'sizeof'.. - std::string sizeof_(const char code[], bool simplify = true, Settings::PlatformType type = Settings::Host) + std::string sizeof_(const char code[], bool simplify = true, Settings::PlatformType type = Settings::Unspecified) { errout.str(""); @@ -977,8 +977,8 @@ private: { const char code[] = ";INT32 i[10];\n" "sizeof(i[0]);\n"; - ASSERT_EQUALS("; INT32 i [ 10 ] ; sizeof ( i [ 0 ] ) ;", sizeof_(code, true, Settings::Host)); - ASSERT_EQUALS("; int i [ 10 ] ; 4 ;", sizeof_(code, true, Settings::Win32)); + ASSERT_EQUALS("; INT32 i [ 10 ] ; sizeof ( i [ 0 ] ) ;", sizeof_(code, true, Settings::Unspecified)); + ASSERT_EQUALS("; int i [ 10 ] ; 4 ;", sizeof_(code, true, Settings::Win32A)); } void sizeof8() @@ -1147,7 +1147,7 @@ private: "compat_ulong_t um ; " "long size ; size = sizeof ( m ) / sizeof ( um ) ;"; - ASSERT_EQUALS(code, tok(code, true, Settings::Win32)); + ASSERT_EQUALS(code, tok(code, true, Settings::Win32A)); } void sizeof11() @@ -2815,7 +2815,7 @@ private: { // Ticket #1997 const char code[] = "void * operator new[](size_t);"; - ASSERT_EQUALS("void * operatornew[] ( long ) ;", tok(code, true, Settings::Win32)); + ASSERT_EQUALS("void * operatornew[] ( long ) ;", tok(code, true, Settings::Win32A)); } ASSERT_EQUALS("; a [ 0 ] ;", tok(";a[0*(*p)];")); diff --git a/test/testtokenize.cpp b/test/testtokenize.cpp index 844e38b36..9c1736df9 100644 --- a/test/testtokenize.cpp +++ b/test/testtokenize.cpp @@ -361,6 +361,8 @@ private: TEST_CASE(removeRedundantCodeAfterReturn); TEST_CASE(platformWin32); + TEST_CASE(platformWin32A); + TEST_CASE(platformWin32W); TEST_CASE(platformWin64); TEST_CASE(platformUnix32); TEST_CASE(platformUnix64); @@ -379,7 +381,7 @@ private: } - std::string tokenizeAndStringify(const char code[], bool simplify = false, bool expand = true, Settings::PlatformType platform = Settings::Host) + std::string tokenizeAndStringify(const char code[], bool simplify = false, bool expand = true, Settings::PlatformType platform = Settings::Unspecified) { errout.str(""); @@ -4014,7 +4016,7 @@ private: const char code[] = "struct foo {\n" " void operator delete(void *obj, size_t sz);\n" "}\n"; - const std::string actual(tokenizeAndStringify(code, false, true, Settings::Win32)); + const std::string actual(tokenizeAndStringify(code, false, true, Settings::Win32A)); const char expected[] = "struct foo {\n" "void operatordelete ( void * obj , unsigned long sz ) ;\n" @@ -5545,37 +5547,37 @@ private: void microsoftMFC() { const char code1[] = "class MyDialog : public CDialog { DECLARE_MESSAGE_MAP() private: CString text; };"; - ASSERT_EQUALS("class MyDialog : public CDialog { private: CString text ; } ;", tokenizeAndStringify(code1,false,true,Settings::Win32)); + ASSERT_EQUALS("class MyDialog : public CDialog { private: CString text ; } ;", tokenizeAndStringify(code1,false,true,Settings::Win32A)); const char code2[] = "class MyDialog : public CDialog { DECLARE_DYNAMIC(MyDialog) private: CString text; };"; - ASSERT_EQUALS("class MyDialog : public CDialog { private: CString text ; } ;", tokenizeAndStringify(code2,false,true,Settings::Win32)); + ASSERT_EQUALS("class MyDialog : public CDialog { private: CString text ; } ;", tokenizeAndStringify(code2,false,true,Settings::Win32A)); const char code3[] = "class MyDialog : public CDialog { DECLARE_DYNCREATE(MyDialog) private: CString text; };"; - ASSERT_EQUALS("class MyDialog : public CDialog { private: CString text ; } ;", tokenizeAndStringify(code3,false,true,Settings::Win32)); + ASSERT_EQUALS("class MyDialog : public CDialog { private: CString text ; } ;", tokenizeAndStringify(code3,false,true,Settings::Win32A)); const char code4[] = "class MyDialog : public CDialog { DECLARE_DYNAMIC_CLASS(MyDialog) private: CString text; };"; - ASSERT_EQUALS("class MyDialog : public CDialog { private: CString text ; } ;", tokenizeAndStringify(code4,false,true,Settings::Win32)); + ASSERT_EQUALS("class MyDialog : public CDialog { private: CString text ; } ;", tokenizeAndStringify(code4,false,true,Settings::Win32A)); } void microsoftMemory() { const char code1[] = "void foo() { int a[10], b[10]; CopyMemory(a, b, sizeof(a)); }"; - ASSERT_EQUALS("void foo ( ) { int a [ 10 ] ; int b [ 10 ] ; memcpy ( a , b , sizeof ( a ) ) ; }", tokenizeAndStringify(code1,false,true,Settings::Win32)); + ASSERT_EQUALS("void foo ( ) { int a [ 10 ] ; int b [ 10 ] ; memcpy ( a , b , sizeof ( a ) ) ; }", tokenizeAndStringify(code1,false,true,Settings::Win32A)); const char code2[] = "void foo() { int a[10]; FillMemory(a, sizeof(a), 255); }"; - ASSERT_EQUALS("void foo ( ) { int a [ 10 ] ; memset ( a , 255 , sizeof ( a ) ) ; }", tokenizeAndStringify(code2,false,true,Settings::Win32)); + ASSERT_EQUALS("void foo ( ) { int a [ 10 ] ; memset ( a , 255 , sizeof ( a ) ) ; }", tokenizeAndStringify(code2,false,true,Settings::Win32A)); const char code3[] = "void foo() { int a[10], b[10]; MoveMemory(a, b, sizeof(a)); }"; - ASSERT_EQUALS("void foo ( ) { int a [ 10 ] ; int b [ 10 ] ; memmove ( a , b , sizeof ( a ) ) ; }", tokenizeAndStringify(code3,false,true,Settings::Win32)); + ASSERT_EQUALS("void foo ( ) { int a [ 10 ] ; int b [ 10 ] ; memmove ( a , b , sizeof ( a ) ) ; }", tokenizeAndStringify(code3,false,true,Settings::Win32A)); const char code4[] = "void foo() { int a[10]; ZeroMemory(a, sizeof(a)); }"; - ASSERT_EQUALS("void foo ( ) { int a [ 10 ] ; memset ( a , 0 , sizeof ( a ) ) ; }", tokenizeAndStringify(code4,false,true,Settings::Win32)); + ASSERT_EQUALS("void foo ( ) { int a [ 10 ] ; memset ( a , 0 , sizeof ( a ) ) ; }", tokenizeAndStringify(code4,false,true,Settings::Win32A)); const char code5[] = "void foo() { ZeroMemory(f(1, g(a, b)), h(i, j(0, 1))); }"; - ASSERT_EQUALS("void foo ( ) { memset ( f ( 1 , g ( a , b ) ) , 0 , h ( i , j ( 0 , 1 ) ) ) ; }", tokenizeAndStringify(code5,false,true,Settings::Win32)); + ASSERT_EQUALS("void foo ( ) { memset ( f ( 1 , g ( a , b ) ) , 0 , h ( i , j ( 0 , 1 ) ) ) ; }", tokenizeAndStringify(code5,false,true,Settings::Win32A)); const char code6[] = "void foo() { FillMemory(f(1, g(a, b)), h(i, j(0, 1)), 255); }"; - ASSERT_EQUALS("void foo ( ) { memset ( f ( 1 , g ( a , b ) ) , 255 , h ( i , j ( 0 , 1 ) ) ) ; }", tokenizeAndStringify(code6,false,true,Settings::Win32)); + ASSERT_EQUALS("void foo ( ) { memset ( f ( 1 , g ( a , b ) ) , 255 , h ( i , j ( 0 , 1 ) ) ) ; }", tokenizeAndStringify(code6,false,true,Settings::Win32A)); } void borland() @@ -6087,7 +6089,40 @@ private: "long S ; " "void * T ;"; - ASSERT_EQUALS(expected, tokenizeAndStringify(code, true, true, Settings::Win32)); + ASSERT_EQUALS(expected, tokenizeAndStringify(code, true, true, Settings::Win32A)); + } + + void platformWin32A() + { + const char code[] = "TCHAR c;" + "PTSTR ptstr;" + "LPTSTR lptstr;" + "PCTSTR pctstr;" + "LPCTSTR lpctstr;" + "void foo() {" + " TCHAR src[10] = _T(\"123456789\");" + " TCHAR dst[10];" + " _tcscpy(dst, src);" + " dst[0] = 0;" + " _tcscat(dst, str);" + "}"; + const char expected[] = "char c ; " + "char * ptstr ; " + "char * lptstr ; " + "const char * pctstr ; " + "const char * lpctstr ; " + "void foo ( ) { " + "char src [ 10 ] = \"123456789\" ; " + "char dst [ 10 ] ; " + "strcpy ( dst , src ) ; " + "dst [ 0 ] = 0 ; " + "strcat ( dst , str ) ; " + "}"; + ASSERT_EQUALS(expected, tokenizeAndStringify(code, false, true, Settings::Win32A)); + } + + void platformWin32W() + { } void platformWin64()