Unlimited macro string literal length and single-quoted strings Available as a patch: http://sourceforge.net/tracker/?func=detail&atid=311005&aid=1598271&group_id=11005 [ 1598271 ] Unlimited macro string literal length, single-quoted strings macroStringLiterals.diff 2006-11-21 String literals are scanned twice, firstly to calculate their space requirements, secondly to read their contents into allocated memory. Separate string literals that follow one another are combined into one, avoiding run-time concatenation of the pieces. Also single-quoted string literals are allowed, within which backslash ('\') has no special meaning (so you can't include a single-quote in a single-quoted string). Note that a double-quoted string can be continued over multiple lines by ending each line but the last with a backslash, like in C. 2006-11-21: Fixed adjacent string literal merging to allow concatenation with "". diff -ur nedit_official nedit_mod diff -ur nedit_official/doc/help.etx nedit_mod/doc/help.etx --- nedit_official/doc/help.etx 2006-10-20 17:44:14.000000000 +0200 +++ nedit_mod/doc/help.etx 2006-11-17 02:58:07.193929600 +0100 @@ -1945,11 +1945,12 @@ Blank lines and comments are also allowed. Comments begin with a "#" and end with a newline, and can appear either on a line by themselves, or at the end - of a statement. + of a statement line. Statements which are too long to fit on a single line may be split across several lines, by placing a backslash "\" character at the end of each line - to be continued. + to be continued. Note that a comment with a backslash at the end is treated + as a continuation in this way too. 3>Data Types @@ -1971,12 +1972,14 @@ 4>Character String Constants - Character string constants are enclosed in double quotes. For example: + Character string constants are enclosed in single or double quotes, but the + start and end quotes must be the same character. For example: a = "a string" - dialog("Hi there!", "OK") + dialog('Hi there!', "OK") - Strings may also include C-language style escape sequences: + A double-quoted string may also include C-language style escape + sequences: \\ Backslash \t Tab \f Form feed \" Double quote \b Backspace \a Alert @@ -1994,11 +1997,11 @@ t_print("a = " a "\n") Other characters can be expressed as backslash-escape sequences in macro - strings. The format is the same as for regular expressions, described in the - paragraphs headed "Octal and Hex Escape Sequences" of the section - "Metacharacters_", except that an octal escape sequence can start with any - octal digit, not just 0, so the single character string "\0033" is the same - as "\33", "\x1B" and "\e" (for an ASCII version of NEdit). + double-quoted strings. The format is the same as for regular expressions, + described in the paragraphs headed "Octal and Hex Escape Sequences" of the + section "Metacharacters_", except that an octal escape sequence can start with + any octal digit, not just 0, so the single character string "\0033" is the + same as "\33", "\x1B" and "\e" (for an ASCII version of NEdit). Note that if you want to define a regular expression in a macro string, you need to "double-up" the backslashes for the metacharacters with @@ -2007,7 +2010,7 @@ (?N(\s|/\*(?n(?:(?!\*/).)*)\*/|//.*\n|\n)+) which matches whitespace or C/C++/Java-style comments, should be written as - a macro string as + a macro double-quoted string as "(?N(\\s|/\\*(?n(?:(?!\\*/).)*)\\*/|//.*\n|\n)+)" @@ -2016,6 +2019,22 @@ also interpret the sequence "\\n" as a newline, although the macro string here would then contain a literal backslash followed by a lowercase `N'.) + Alternatively, if you don't need special escapes or a single quote + (apostrophe) in your string (true for this example), just turn the expression + into a single-quoted string, as + + '(?N(\s|/\*(?n(?:(?!\*/).)*)\*/|//.*\n|\n)+)' + + Neighboring string literals (separated by whitespace or line continuations) + are combined, as if by the concatenation operation before use. For example + + "The backslash '" '\' "' is an " \ + 'escape only in "double-quoted" strings' "\n" + + is treated as a single string ending with a newline character, looking like + + The backslash '\' is an escape only in "double-quoted" strings + 3>Variables diff -ur nedit_official/source/parse.y nedit_mod/source/parse.y --- nedit_official/source/parse.y 2005-02-16 04:44:16.000000000 +0100 +++ nedit_mod/source/parse.y 2006-11-17 01:58:47.845838400 +0100 @@ -46,6 +46,7 @@ static int follow2(char expect1, int yes1, char expect2, int yes2, int no); static int follow_non_whitespace(char expect, int yes, int no); static Symbol *matchesActionRoutine(char **inPtr); +static int scanString(void); static char *ErrMsg; static char *InPtr; @@ -477,19 +478,8 @@ return prog; } - -static int yylex(void) -{ - int i, len; - Symbol *s; - static DataValue value = {NO_TAG, {0}}; - static char escape[] = "\\\"ntbrfave"; -#ifdef EBCDIC_CHARSET - static char replace[] = "\\\"\n\t\b\r\f\a\v\x27"; /* EBCDIC escape */ -#else - static char replace[] = "\\\"\n\t\b\r\f\a\v\x1B"; /* ASCII escape */ -#endif - +static char skipWhitespace(void) +{ /* skip whitespace, backslash-newline combinations, and comments, which are all considered whitespace */ for (;;) { @@ -497,7 +487,8 @@ InPtr += 2; else if (*InPtr == ' ' || *InPtr == '\t') InPtr++; - else if (*InPtr == '#') + else if (*InPtr == '#') { + InPtr++; while (*InPtr != '\n' && *InPtr != '\0') { /* Comments stop at escaped newlines */ if (*InPtr == '\\' && *(InPtr + 1) == '\n') { @@ -505,10 +496,21 @@ break; } InPtr++; - } else + } + } + else break; } + return *InPtr; +} +static int yylex(void) +{ + int len; + Symbol *s; + static DataValue value = {NO_TAG, {0}}; + + skipWhitespace(); /* return end of input at the end of the string */ if (*InPtr == '\0') { @@ -567,115 +569,10 @@ return SYMBOL; } - /* Process quoted strings with embedded escape sequences: - For backslashes we recognise hexadecimal values with initial 'x' such - as "\x1B"; octal value (upto 3 oct digits with a possible leading zero) - such as "\33", "\033" or "\0033", and the C escapes: \", \', \n, \t, \b, - \r, \f, \a, \v, and the added \e for the escape character, as for REs. - Disallow hex/octal zero values (NUL): instead ignore the introductory - backslash, eg "\x0xyz" becomes "x0xyz" and "\0000hello" becomes - "0000hello". */ - - if (*InPtr == '\"') { - char string[MAX_STRING_CONST_LEN], *p = string; - char *backslash; - InPtr++; - while (*InPtr != '\0' && *InPtr != '\"' && *InPtr != '\n') { - if (p >= string + MAX_STRING_CONST_LEN) { - InPtr++; - continue; - } - if (*InPtr == '\\') { - backslash = InPtr; - InPtr++; - if (*InPtr == '\n') { - InPtr++; - continue; - } - if (*InPtr == 'x') { - /* a hex introducer */ - int hexValue = 0; - const char *hexDigits = "0123456789abcdef"; - const char *hexD; - InPtr++; - if (*InPtr == '\0' || - (hexD = strchr(hexDigits, tolower(*InPtr))) == NULL) { - *p++ = 'x'; - } - else { - hexValue = hexD - hexDigits; - InPtr++; - /* now do we have another digit? only accept one more */ - if (*InPtr != '\0' && - (hexD = strchr(hexDigits,tolower(*InPtr))) != NULL){ - hexValue = hexD - hexDigits + (hexValue << 4); - InPtr++; - } - if (hexValue != 0) { - *p++ = (char)hexValue; - } - else { - InPtr = backslash + 1; /* just skip the backslash */ - } - } - continue; - } - /* the RE documentation requires \0 as the octal introducer; - here you can start with any octal digit, but you are only - allowed up to three (or four if the first is '0'). */ - if ('0' <= *InPtr && *InPtr <= '7') { - if (*InPtr == '0') { - InPtr++; /* octal introducer: don't count this digit */ - } - if ('0' <= *InPtr && *InPtr <= '7') { - /* treat as octal - first digit */ - char octD = *InPtr++; - int octValue = octD - '0'; - if ('0' <= *InPtr && *InPtr <= '7') { - /* second digit */ - octD = *InPtr++; - octValue = (octValue << 3) + octD - '0'; - /* now do we have another digit? can we add it? - if value is going to be too big for char (greater - than 0377), stop converting now before adding the - third digit */ - if ('0' <= *InPtr && *InPtr <= '7' && - octValue <= 037) { - /* third digit is acceptable */ - octD = *InPtr++; - octValue = (octValue << 3) + octD - '0'; - } - } - if (octValue != 0) { - *p++ = (char)octValue; - } - else { - InPtr = backslash + 1; /* just skip the backslash */ - } - } - else { /* \0 followed by non-digits: go back to 0 */ - InPtr = backslash + 1; /* just skip the backslash */ - } - continue; - } - for (i=0; escape[i]!='\0'; i++) { - if (escape[i] == *InPtr) { - *p++ = replace[i]; - InPtr++; - break; - } - } - /* if we get here, we didn't recognise the character after - the backslash: just copy it next time round the loop */ - } - else { - *p++= *InPtr++; - } - } - *p = '\0'; - InPtr++; - yylval.sym = InstallStringConstSymbol(string); - return STRING; + /* Process quoted strings */ + + if (*InPtr == '\"' || *InPtr == '\'') { + return scanString(); } /* process remaining two character tokens or return single char as token */ @@ -770,6 +667,177 @@ } /* +** Process quoted string literals. These can be in single or double quotes. +** A sequence of string literals separated by whitespace (see skipWhitespace()) +** are read as a single string. +** +** Double-quoted string literals allow embedded escape sequences: +** For backslashes we recognise hexadecimal values with initial 'x' such +** as "\x1B"; octal value (upto 3 oct digits with a possible leading zero) +** such as "\33", "\033" or "\0033", and the C escapes: \", \', \n, \t, \b, +** \r, \f, \a, \v, and the added \e for the escape character, as for REs. +** We disallow hex/octal zero values (NUL): instead ignore the introductory +** backslash, eg "\x0xyz" becomes "x0xyz" and "\0000hello" becomes "0000hello". +** An escaped newline is elided, and the string content continues on the next +** source line. +*/ +static int scanString(void) +{ +# define SCANSTRING_WRITE_TO_STRING(p, len, val) \ + do { char mc = (val); if (p) { *p++ = mc; } else { ++len; } } while (0) + + /* scan the string twice: once to get its size, then again to build it */ + char *startPtr = InPtr; + char *p = NULL, *string = NULL; + int len, scan, i; + char stopper, first_stopper = *startPtr; + char *backslash; + int handleBackslash; + + static char escape[] = "\\\"ntbrfave"; +#ifdef EBCDIC_CHARSET + static char replace[] = "\\\"\n\t\b\r\f\a\v\x27"; /* EBCDIC escape */ +#else + static char replace[] = "\\\"\n\t\b\r\f\a\v\x1B"; /* ASCII escape */ +#endif + + if (first_stopper != '\"' && first_stopper != '\'') + return yyerror("expected a string"); + + for (scan = 0; scan < 2; ++scan) + { + InPtr = startPtr; + stopper = first_stopper; + handleBackslash = (stopper == '\"'); + len = 0; + InPtr++; + while (*InPtr != '\0' && *InPtr != '\n') { + if (*InPtr == stopper) { + char *endPtr = InPtr++; + skipWhitespace(); + /* is this followed by another string literal? */ + if (*InPtr == '\"' || *InPtr == '\'') { + stopper = *InPtr++; /* add it to the end of the first */ + handleBackslash = (stopper == '\"'); + } + else { + InPtr = endPtr; /* no further string: restore position */ + break; + } + } + else if (handleBackslash && *InPtr == '\\') { + backslash = InPtr; + InPtr++; + if (*InPtr == '\n') { /* allows newline to be skipped */ + InPtr++; + continue; + } + if (*InPtr == 'x') { + /* a hex introducer */ + int hexValue = 0; + const char *hexDigits = "0123456789abcdef"; + const char *hexD; + InPtr++; + if (*InPtr == '\0') + break; + if ((hexD = strchr(hexDigits, tolower(*InPtr))) == NULL) { + SCANSTRING_WRITE_TO_STRING(p, len, 'x'); + } + else { + hexValue = hexD - hexDigits; + InPtr++; + if (*InPtr == '\0') + break; + /* now do we have another digit? only accept one more */ + if ((hexD = strchr(hexDigits,tolower(*InPtr))) != NULL){ + hexValue = hexD - hexDigits + (hexValue << 4); + InPtr++; + } + if (hexValue != 0) { + SCANSTRING_WRITE_TO_STRING(p, len, (char)hexValue); + } + else { + InPtr = backslash + 1; /* just skip the backslash */ + } + } + continue; + } + /* the RE documentation requires \0 as the octal introducer; + here you can start with any octal digit, but you are only + allowed up to three (or four if the first is '0'). */ + if ('0' <= *InPtr && *InPtr <= '7') { + if (*InPtr == '0') { + InPtr++; /* octal introducer: don't count this digit */ + } + if ('0' <= *InPtr && *InPtr <= '7') { + /* treat as octal - first digit */ + char octD = *InPtr++; + int octValue = octD - '0'; + if ('0' <= *InPtr && *InPtr <= '7') { + /* second digit */ + octD = *InPtr++; + octValue = (octValue << 3) + octD - '0'; + /* now do we have another digit? can we add it? + if value is going to be too big for char (greater + than 0377), stop converting now before adding the + third digit */ + if ('0' <= *InPtr && *InPtr <= '7' && + octValue <= 037) { + /* third digit is acceptable */ + octD = *InPtr++; + octValue = (octValue << 3) + octD - '0'; + } + } + if (octValue != 0) { + SCANSTRING_WRITE_TO_STRING(p, len, (char)octValue); + } + else { + InPtr = backslash + 1; /* just skip the backslash */ + } + } + else { /* \0 followed by non-digits: go back to 0 */ + InPtr = backslash + 1; /* just skip the backslash */ + } + continue; + } + /* check for a valid c-style escape character */ + for (i = 0; escape[i] != '\0'; i++) { + if (escape[i] == *InPtr) { + SCANSTRING_WRITE_TO_STRING(p, len, replace[i]); + InPtr++; + break; + } + } + /* if we get here, we didn't recognise the character after + the backslash: just copy it next time round the loop */ + } + else { + SCANSTRING_WRITE_TO_STRING(p, len, *InPtr++); + } + } + /* terminate the string content */ + SCANSTRING_WRITE_TO_STRING(p, len, '\0'); + if (*InPtr == stopper) { + if (!p) { + /* this was the size measurement and validation */ + p = string = AllocString(len); + } + else { + /* OK: string now contains our string text */ + InPtr++; /* skip past stopper */ + yylval.sym = InstallStringConstSymbol(string); + return STRING; + } + } + else { + /* failure: end quote doesn't match start quote */ + break; + } + } + return yyerror("unterminated string"); +} + +/* ** Called by yacc to report errors (just stores for returning when ** parsing is aborted. The error token action is to immediate abort ** parsing, so this message is immediately reported to the caller