/* xgettext Python backend. Copyright (C) 2002-2003, 2005-2009 Free Software Foundation, Inc. This file was written by Bruno Haible , 2002. This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 3 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program. If not, see . */ #ifdef HAVE_CONFIG_H # include "config.h" #endif /* Specification. */ #include "x-python.h" #include #include #include #include #include #include #include "message.h" #include "xgettext.h" #include "error.h" #include "error-progname.h" #include "progname.h" #include "basename.h" #include "xerror.h" #include "xvasprintf.h" #include "xalloc.h" #include "c-strstr.h" #include "c-ctype.h" #include "po-charset.h" #include "uniname.h" #include "unistr.h" #include "gettext.h" #define _(s) gettext(s) #define max(a,b) ((a) > (b) ? (a) : (b)) #define SIZEOF(a) (sizeof(a) / sizeof(a[0])) /* The Python syntax is defined in the Python Reference Manual /usr/share/doc/packages/python/html/ref/index.html. See also Python-2.0/Parser/tokenizer.c, Python-2.0/Python/compile.c, Python-2.0/Objects/unicodeobject.c. */ /* ====================== Keyword set customization. ====================== */ /* If true extract all strings. */ static bool extract_all = false; static hash_table keywords; static bool default_keywords = true; void x_python_extract_all () { extract_all = true; } void x_python_keyword (const char *name) { if (name == NULL) default_keywords = false; else { const char *end; struct callshape shape; const char *colon; if (keywords.table == NULL) hash_init (&keywords, 100); split_keywordspec (name, &end, &shape); /* The characters between name and end should form a valid C identifier. A colon means an invalid parse in split_keywordspec(). */ colon = strchr (name, ':'); if (colon == NULL || colon >= end) insert_keyword_callshape (&keywords, name, end - name, &shape); } } /* Finish initializing the keywords hash table. Called after argument processing, before each file is processed. */ static void init_keywords () { if (default_keywords) { /* When adding new keywords here, also update the documentation in xgettext.texi! */ x_python_keyword ("gettext"); x_python_keyword ("ugettext"); x_python_keyword ("dgettext:2"); x_python_keyword ("ngettext:1,2"); x_python_keyword ("ungettext:1,2"); x_python_keyword ("dngettext:2,3"); x_python_keyword ("_"); default_keywords = false; } } void init_flag_table_python () { xgettext_record_flag ("gettext:1:pass-python-format"); xgettext_record_flag ("ugettext:1:pass-python-format"); xgettext_record_flag ("dgettext:2:pass-python-format"); xgettext_record_flag ("ngettext:1:pass-python-format"); xgettext_record_flag ("ngettext:2:pass-python-format"); xgettext_record_flag ("ungettext:1:pass-python-format"); xgettext_record_flag ("ungettext:2:pass-python-format"); xgettext_record_flag ("dngettext:2:pass-python-format"); xgettext_record_flag ("dngettext:3:pass-python-format"); xgettext_record_flag ("_:1:pass-python-format"); /* xgettext_record_flag ("%:1:python-format"); // % is an infix operator! */ } /* ======================== Reading of characters. ======================== */ /* Real filename, used in error messages about the input file. */ static const char *real_file_name; /* Logical filename and line number, used to label the extracted messages. */ static char *logical_file_name; static int line_number; /* The input file stream. */ static FILE *fp; /* 1. line_number handling. */ /* Maximum used, roughly a safer MB_LEN_MAX. */ #define MAX_PHASE1_PUSHBACK 16 static unsigned char phase1_pushback[MAX_PHASE1_PUSHBACK]; static int phase1_pushback_length; /* Read the next single byte from the input file. */ static int phase1_getc () { int c; if (phase1_pushback_length) c = phase1_pushback[--phase1_pushback_length]; else { c = getc (fp); if (c == EOF) { if (ferror (fp)) error (EXIT_FAILURE, errno, _("error while reading \"%s\""), real_file_name); return EOF; } } if (c == '\n') ++line_number; return c; } /* Supports MAX_PHASE1_PUSHBACK characters of pushback. */ static void phase1_ungetc (int c) { if (c != EOF) { if (c == '\n') --line_number; if (phase1_pushback_length == SIZEOF (phase1_pushback)) abort (); phase1_pushback[phase1_pushback_length++] = c; } } /* Phase 2: Conversion to Unicode. This is done early because PEP 0263 specifies that conversion to Unicode conceptually occurs before tokenization. A test case where it matters is with encodings like BIG5: when a double-byte character ending in 0x5C is followed by '\' or 'u0021', the tokenizer must not treat the second half of the double-byte character as a backslash. */ /* End-of-file indicator for functions returning an UCS-4 character. */ #define UEOF -1 static lexical_context_ty lexical_context; static int phase2_pushback[max (9, UNINAME_MAX + 3)]; static int phase2_pushback_length; /* Read the next Unicode UCS-4 character from the input file. */ static int phase2_getc () { if (phase2_pushback_length) return phase2_pushback[--phase2_pushback_length]; if (xgettext_current_source_encoding == po_charset_ascii) { int c = phase1_getc (); if (c == EOF) return UEOF; if (!c_isascii (c)) { multiline_error (xstrdup (""), xasprintf ("%s\n%s\n", non_ascii_error_message (lexical_context, real_file_name, line_number), _("\ Please specify the source encoding through --from-code or through a comment\n\ as specified in http://www.python.org/peps/pep-0263.html.\n"))); exit (EXIT_FAILURE); } return c; } else if (xgettext_current_source_encoding != po_charset_utf8) { #if HAVE_ICONV /* Use iconv on an increasing number of bytes. Read only as many bytes through phase1_getc as needed. This is needed to give reasonable interactive behaviour when fp is connected to an interactive tty. */ unsigned char buf[MAX_PHASE1_PUSHBACK]; size_t bufcount; int c = phase1_getc (); if (c == EOF) return UEOF; buf[0] = (unsigned char) c; bufcount = 1; for (;;) { unsigned char scratchbuf[6]; const char *inptr = (const char *) &buf[0]; size_t insize = bufcount; char *outptr = (char *) &scratchbuf[0]; size_t outsize = sizeof (scratchbuf); size_t res = iconv (xgettext_current_source_iconv, (ICONV_CONST char **) &inptr, &insize, &outptr, &outsize); /* We expect that a character has been produced if and only if some input bytes have been consumed. */ if ((insize < bufcount) != (outsize < sizeof (scratchbuf))) abort (); if (outsize == sizeof (scratchbuf)) { /* No character has been produced. Must be an error. */ if (res != (size_t)(-1)) abort (); if (errno == EILSEQ) { /* An invalid multibyte sequence was encountered. */ multiline_error (xstrdup (""), xasprintf (_("\ %s:%d: Invalid multibyte sequence.\n\ Please specify the correct source encoding through --from-code or through a\n\ comment as specified in http://www.python.org/peps/pep-0263.html.\n"), real_file_name, line_number)); exit (EXIT_FAILURE); } else if (errno == EINVAL) { /* An incomplete multibyte character. */ int c; if (bufcount == MAX_PHASE1_PUSHBACK) { /* An overlong incomplete multibyte sequence was encountered. */ multiline_error (xstrdup (""), xasprintf (_("\ %s:%d: Long incomplete multibyte sequence.\n\ Please specify the correct source encoding through --from-code or through a\n\ comment as specified in http://www.python.org/peps/pep-0263.html.\n"), real_file_name, line_number)); exit (EXIT_FAILURE); } /* Read one more byte and retry iconv. */ c = phase1_getc (); if (c == EOF) { multiline_error (xstrdup (""), xasprintf (_("\ %s:%d: Incomplete multibyte sequence at end of file.\n\ Please specify the correct source encoding through --from-code or through a\n\ comment as specified in http://www.python.org/peps/pep-0263.html.\n"), real_file_name, line_number)); exit (EXIT_FAILURE); } if (c == '\n') { multiline_error (xstrdup (""), xasprintf (_("\ %s:%d: Incomplete multibyte sequence at end of line.\n\ Please specify the correct source encoding through --from-code or through a\n\ comment as specified in http://www.python.org/peps/pep-0263.html.\n"), real_file_name, line_number - 1)); exit (EXIT_FAILURE); } buf[bufcount++] = (unsigned char) c; } else error (EXIT_FAILURE, errno, _("%s:%d: iconv failure"), real_file_name, line_number); } else { size_t outbytes = sizeof (scratchbuf) - outsize; size_t bytes = bufcount - insize; ucs4_t uc; /* We expect that one character has been produced. */ if (bytes == 0) abort (); if (outbytes == 0) abort (); /* Push back the unused bytes. */ while (insize > 0) phase1_ungetc (buf[--insize]); /* Convert the character from UTF-8 to UCS-4. */ if (u8_mbtouc (&uc, scratchbuf, outbytes) < outbytes) { /* scratchbuf contains an out-of-range Unicode character (> 0x10ffff). */ multiline_error (xstrdup (""), xasprintf (_("\ %s:%d: Invalid multibyte sequence.\n\ Please specify the source encoding through --from-code or through a comment\n\ as specified in http://www.python.org/peps/pep-0263.html.\n"), real_file_name, line_number)); exit (EXIT_FAILURE); } return uc; } } #else /* If we don't have iconv(), the only supported values for xgettext_global_source_encoding and thus also for xgettext_current_source_encoding are ASCII and UTF-8. */ abort (); #endif } else { /* Read an UTF-8 encoded character. */ unsigned char buf[6]; unsigned int count; int c; ucs4_t uc; c = phase1_getc (); if (c == EOF) return UEOF; buf[0] = c; count = 1; if (buf[0] >= 0xc0) { c = phase1_getc (); if (c == EOF) return UEOF; buf[1] = c; count = 2; } if (buf[0] >= 0xe0 && ((buf[1] ^ 0x80) < 0x40)) { c = phase1_getc (); if (c == EOF) return UEOF; buf[2] = c; count = 3; } if (buf[0] >= 0xf0 && ((buf[1] ^ 0x80) < 0x40) && ((buf[2] ^ 0x80) < 0x40)) { c = phase1_getc (); if (c == EOF) return UEOF; buf[3] = c; count = 4; } if (buf[0] >= 0xf8 && ((buf[1] ^ 0x80) < 0x40) && ((buf[2] ^ 0x80) < 0x40) && ((buf[3] ^ 0x80) < 0x40)) { c = phase1_getc (); if (c == EOF) return UEOF; buf[4] = c; count = 5; } if (buf[0] >= 0xfc && ((buf[1] ^ 0x80) < 0x40) && ((buf[2] ^ 0x80) < 0x40) && ((buf[3] ^ 0x80) < 0x40) && ((buf[4] ^ 0x80) < 0x40)) { c = phase1_getc (); if (c == EOF) return UEOF; buf[5] = c; count = 6; } u8_mbtouc (&uc, buf, count); return uc; } } /* Supports max (9, UNINAME_MAX + 3) pushback characters. */ static void phase2_ungetc (int c) { if (c != UEOF) { if (phase2_pushback_length == SIZEOF (phase2_pushback)) abort (); phase2_pushback[phase2_pushback_length++] = c; } } /* ========================= Accumulating strings. ======================== */ /* A string buffer type that allows appending Unicode characters. Returns the entire string in UTF-8 encoding. */ struct unicode_string_buffer { /* The part of the string that has already been converted to UTF-8. */ char *utf8_buffer; size_t utf8_buflen; size_t utf8_allocated; }; /* Initialize a 'struct unicode_string_buffer' to empty. */ static inline void init_unicode_string_buffer (struct unicode_string_buffer *bp) { bp->utf8_buffer = NULL; bp->utf8_buflen = 0; bp->utf8_allocated = 0; } /* Auxiliary function: Ensure count more bytes are available in bp->utf8. */ static inline void unicode_string_buffer_append_unicode_grow (struct unicode_string_buffer *bp, size_t count) { if (bp->utf8_buflen + count > bp->utf8_allocated) { size_t new_allocated = 2 * bp->utf8_allocated + 10; if (new_allocated < bp->utf8_buflen + count) new_allocated = bp->utf8_buflen + count; bp->utf8_allocated = new_allocated; bp->utf8_buffer = xrealloc (bp->utf8_buffer, new_allocated); } } /* Auxiliary function: Append a Unicode character to bp->utf8. uc must be < 0x110000. */ static inline void unicode_string_buffer_append_unicode (struct unicode_string_buffer *bp, unsigned int uc) { unsigned char utf8buf[6]; int count = u8_uctomb (utf8buf, uc, 6); if (count < 0) /* The caller should have ensured that uc is not out-of-range. */ abort (); unicode_string_buffer_append_unicode_grow (bp, count); memcpy (bp->utf8_buffer + bp->utf8_buflen, utf8buf, count); bp->utf8_buflen += count; } /* Return the string buffer's contents. */ static char * unicode_string_buffer_result (struct unicode_string_buffer *bp) { /* NUL-terminate it. */ unicode_string_buffer_append_unicode_grow (bp, 1); bp->utf8_buffer[bp->utf8_buflen] = '\0'; /* Return it. */ return bp->utf8_buffer; } /* Free the memory pointed to by a 'struct unicode_string_buffer'. */ static inline void free_unicode_string_buffer (struct unicode_string_buffer *bp) { free (bp->utf8_buffer); } /* ======================== Accumulating comments. ======================== */ /* Accumulating a single comment line. */ static struct unicode_string_buffer comment_buffer; static inline void comment_start () { lexical_context = lc_comment; comment_buffer.utf8_buflen = 0; } static inline bool comment_at_start () { return (comment_buffer.utf8_buflen == 0); } static inline void comment_add (int c) { unicode_string_buffer_append_unicode (&comment_buffer, c); } static inline const char * comment_line_end () { char *buffer = unicode_string_buffer_result (&comment_buffer); size_t buflen = strlen (buffer); while (buflen >= 1 && (buffer[buflen - 1] == ' ' || buffer[buflen - 1] == '\t')) --buflen; buffer[buflen] = '\0'; savable_comment_add (buffer); lexical_context = lc_outside; return buffer; } /* These are for tracking whether comments count as immediately before keyword. */ static int last_comment_line; static int last_non_comment_line; /* ======================== Recognizing comments. ======================== */ /* Recognizing the "coding" comment. As specified in PEP 0263, it takes the form "coding" [":"|"="] {alphanumeric or "-" or "_" or "*"}* or "set" "fileencoding" "=" {alphanumeric or "-" or "_" or "*"}* and is located in a comment in a line that - is either the first or second line, - is not a continuation line, - in the first form, contains no other tokens except this comment. */ /* Canonicalized encoding name for the current input file. */ static const char *xgettext_current_file_source_encoding; #if HAVE_ICONV /* Converter from xgettext_current_file_source_encoding to UTF-8 (except from ASCII or UTF-8, when this conversion is a no-op). */ static iconv_t xgettext_current_file_source_iconv; #endif static inline void set_current_file_source_encoding (const char *canon_encoding) { xgettext_current_file_source_encoding = canon_encoding; if (xgettext_current_file_source_encoding != po_charset_ascii && xgettext_current_file_source_encoding != po_charset_utf8) { #if HAVE_ICONV iconv_t cd; /* Avoid glibc-2.1 bug with EUC-KR. */ # if (__GLIBC__ - 0 == 2 && __GLIBC_MINOR__ - 0 <= 1) && !defined _LIBICONV_VERSION if (strcmp (xgettext_current_file_source_encoding, "EUC-KR") == 0) cd = (iconv_t)(-1); else # endif cd = iconv_open (po_charset_utf8, xgettext_current_file_source_encoding); if (cd == (iconv_t)(-1)) error_at_line (EXIT_FAILURE, 0, logical_file_name, line_number - 1, _("\ Cannot convert from \"%s\" to \"%s\". %s relies on iconv(), \ and iconv() does not support this conversion."), xgettext_current_file_source_encoding, po_charset_utf8, basename (program_name)); xgettext_current_file_source_iconv = cd; #else error_at_line (EXIT_FAILURE, 0, logical_file_name, line_number - 1, _("\ Cannot convert from \"%s\" to \"%s\". %s relies on iconv(). \ This version was built without iconv()."), xgettext_global_source_encoding, po_charset_utf8, basename (program_name)); #endif } xgettext_current_source_encoding = xgettext_current_file_source_encoding; #if HAVE_ICONV xgettext_current_source_iconv = xgettext_current_file_source_iconv; #endif } static inline void try_to_extract_coding (const char *comment) { const char *p = c_strstr (comment, "coding"); if (p != NULL) { p += 6; if (*p == ':' || *p == '=') { p++; while (*p == ' ' || *p == '\t') p++; { const char *encoding_start = p; while (c_isalnum (*p) || *p == '-' || *p == '_' || *p == '.') p++; { const char *encoding_end = p; if (encoding_end > encoding_start) { /* Extract the encoding string. */ size_t encoding_len = encoding_end - encoding_start; char *encoding = XNMALLOC (encoding_len + 1, char); memcpy (encoding, encoding_start, encoding_len); encoding[encoding_len] = '\0'; { /* Canonicalize it. */ const char *canon_encoding = po_charset_canonicalize (encoding); if (canon_encoding == NULL) { error_at_line (0, 0, logical_file_name, line_number - 1, _("\ Unknown encoding \"%s\". Proceeding with ASCII instead."), encoding); canon_encoding = po_charset_ascii; } /* Activate it. */ set_current_file_source_encoding (canon_encoding); } free (encoding); } } } } } } /* Tracking whether the current line is a continuation line or contains a non-blank character. */ static bool continuation_or_nonblank_line = false; /* Phase 3: Outside strings, replace backslash-newline with nothing and a comment with nothing. */ static int phase3_getc () { int c; for (;;) { c = phase2_getc (); if (c == '\\') { c = phase2_getc (); if (c != '\n') { phase2_ungetc (c); /* This shouldn't happen usually, because "A backslash is illegal elsewhere on a line outside a string literal." */ return '\\'; } /* Eat backslash-newline. */ continuation_or_nonblank_line = true; } else if (c == '#') { /* Eat a comment. */ const char *comment; last_comment_line = line_number; comment_start (); for (;;) { c = phase2_getc (); if (c == UEOF || c == '\n') break; /* We skip all leading white space, but not EOLs. */ if (!(comment_at_start () && (c == ' ' || c == '\t'))) comment_add (c); } comment = comment_line_end (); if (line_number - 1 <= 2 && !continuation_or_nonblank_line) try_to_extract_coding (comment); continuation_or_nonblank_line = false; return c; } else { if (c == '\n') continuation_or_nonblank_line = false; else if (!(c == ' ' || c == '\t' || c == '\f')) continuation_or_nonblank_line = true; return c; } } } /* Supports only one pushback character. */ static void phase3_ungetc (int c) { phase2_ungetc (c); } /* ========================= Accumulating strings. ======================== */ /* Return value of phase7_getuc when EOF is reached. */ #define P7_EOF (-1) #define P7_STRING_END (-2) /* Convert an UTF-16 or UTF-32 code point to a return value that can be distinguished from a single-byte return value. */ #define UNICODE(code) (0x100 + (code)) /* Test a return value of phase7_getuc whether it designates an UTF-16 or UTF-32 code point. */ #define IS_UNICODE(p7_result) ((p7_result) >= 0x100) /* Extract the UTF-16 or UTF-32 code of a return value that satisfies IS_UNICODE. */ #define UNICODE_VALUE(p7_result) ((p7_result) - 0x100) /* A string buffer type that allows appending bytes (in the xgettext_current_source_encoding) or Unicode characters. Returns the entire string in UTF-8 encoding. */ struct mixed_string_buffer { /* The part of the string that has already been converted to UTF-8. */ char *utf8_buffer; size_t utf8_buflen; size_t utf8_allocated; /* The first half of an UTF-16 surrogate character. */ unsigned short utf16_surr; /* The part of the string that is still in the source encoding. */ char *curr_buffer; size_t curr_buflen; size_t curr_allocated; /* The lexical context. Used only for error message purposes. */ lexical_context_ty lcontext; }; /* Initialize a 'struct mixed_string_buffer' to empty. */ static inline void init_mixed_string_buffer (struct mixed_string_buffer *bp, lexical_context_ty lcontext) { bp->utf8_buffer = NULL; bp->utf8_buflen = 0; bp->utf8_allocated = 0; bp->utf16_surr = 0; bp->curr_buffer = NULL; bp->curr_buflen = 0; bp->curr_allocated = 0; bp->lcontext = lcontext; } /* Auxiliary function: Append a byte to bp->curr. */ static inline void mixed_string_buffer_append_byte (struct mixed_string_buffer *bp, unsigned char c) { if (bp->curr_buflen == bp->curr_allocated) { bp->curr_allocated = 2 * bp->curr_allocated + 10; bp->curr_buffer = xrealloc (bp->curr_buffer, bp->curr_allocated); } bp->curr_buffer[bp->curr_buflen++] = c; } /* Auxiliary function: Ensure count more bytes are available in bp->utf8. */ static inline void mixed_string_buffer_append_unicode_grow (struct mixed_string_buffer *bp, size_t count) { if (bp->utf8_buflen + count > bp->utf8_allocated) { size_t new_allocated = 2 * bp->utf8_allocated + 10; if (new_allocated < bp->utf8_buflen + count) new_allocated = bp->utf8_buflen + count; bp->utf8_allocated = new_allocated; bp->utf8_buffer = xrealloc (bp->utf8_buffer, new_allocated); } } /* Auxiliary function: Append a Unicode character to bp->utf8. uc must be < 0x110000. */ static inline void mixed_string_buffer_append_unicode (struct mixed_string_buffer *bp, ucs4_t uc) { unsigned char utf8buf[6]; int count = u8_uctomb (utf8buf, uc, 6); if (count < 0) /* The caller should have ensured that uc is not out-of-range. */ abort (); mixed_string_buffer_append_unicode_grow (bp, count); memcpy (bp->utf8_buffer + bp->utf8_buflen, utf8buf, count); bp->utf8_buflen += count; } /* Auxiliary function: Flush bp->utf16_surr into bp->utf8_buffer. */ static inline void mixed_string_buffer_flush_utf16_surr (struct mixed_string_buffer *bp) { if (bp->utf16_surr != 0) { /* A half surrogate is invalid, therefore use U+FFFD instead. */ mixed_string_buffer_append_unicode (bp, 0xfffd); bp->utf16_surr = 0; } } /* Auxiliary function: Flush bp->curr_buffer into bp->utf8_buffer. */ static inline void mixed_string_buffer_flush_curr_buffer (struct mixed_string_buffer *bp, int lineno) { if (bp->curr_buflen > 0) { char *curr; size_t count; mixed_string_buffer_append_byte (bp, '\0'); /* Convert from the source encoding to UTF-8. */ curr = from_current_source_encoding (bp->curr_buffer, bp->lcontext, logical_file_name, lineno); /* Append it to bp->utf8_buffer. */ count = strlen (curr); mixed_string_buffer_append_unicode_grow (bp, count); memcpy (bp->utf8_buffer + bp->utf8_buflen, curr, count); bp->utf8_buflen += count; if (curr != bp->curr_buffer) free (curr); bp->curr_buflen = 0; } } /* Append a character or Unicode character to a 'struct mixed_string_buffer'. */ static void mixed_string_buffer_append (struct mixed_string_buffer *bp, int c) { if (IS_UNICODE (c)) { /* Append a Unicode character. */ /* Switch from multibyte character mode to Unicode character mode. */ mixed_string_buffer_flush_curr_buffer (bp, line_number); /* Test whether this character and the previous one form a Unicode surrogate character pair. */ if (bp->utf16_surr != 0 && (c >= UNICODE (0xdc00) && c < UNICODE (0xe000))) { unsigned short utf16buf[2]; ucs4_t uc; utf16buf[0] = bp->utf16_surr; utf16buf[1] = UNICODE_VALUE (c); if (u16_mbtouc (&uc, utf16buf, 2) != 2) abort (); mixed_string_buffer_append_unicode (bp, uc); bp->utf16_surr = 0; } else { mixed_string_buffer_flush_utf16_surr (bp); if (c >= UNICODE (0xd800) && c < UNICODE (0xdc00)) bp->utf16_surr = UNICODE_VALUE (c); else if (c >= UNICODE (0xdc00) && c < UNICODE (0xe000)) { /* A half surrogate is invalid, therefore use U+FFFD instead. */ mixed_string_buffer_append_unicode (bp, 0xfffd); } else mixed_string_buffer_append_unicode (bp, UNICODE_VALUE (c)); } } else { /* Append a single byte. */ /* Switch from Unicode character mode to multibyte character mode. */ mixed_string_buffer_flush_utf16_surr (bp); /* When a newline is seen, convert the accumulated multibyte sequence. This ensures a correct line number in the error message in case of a conversion error. The "- 1" is to account for the newline. */ if (c == '\n') mixed_string_buffer_flush_curr_buffer (bp, line_number - 1); mixed_string_buffer_append_byte (bp, (unsigned char) c); } } /* Return the string buffer's contents. */ static char * mixed_string_buffer_result (struct mixed_string_buffer *bp) { /* Flush all into bp->utf8_buffer. */ mixed_string_buffer_flush_utf16_surr (bp); mixed_string_buffer_flush_curr_buffer (bp, line_number); /* NUL-terminate it. */ mixed_string_buffer_append_unicode_grow (bp, 1); bp->utf8_buffer[bp->utf8_buflen] = '\0'; /* Return it. */ return bp->utf8_buffer; } /* Free the memory pointed to by a 'struct mixed_string_buffer'. */ static inline void free_mixed_string_buffer (struct mixed_string_buffer *bp) { free (bp->utf8_buffer); free (bp->curr_buffer); } /* ========================== Reading of tokens. ========================== */ enum token_type_ty { token_type_eof, token_type_lparen, /* ( */ token_type_rparen, /* ) */ token_type_comma, /* , */ token_type_lbracket, /* [ */ token_type_rbracket, /* ] */ token_type_string, /* "abc", 'abc', """abc""", '''abc''' */ token_type_symbol, /* symbol, number */ token_type_other /* misc. operator */ }; typedef enum token_type_ty token_type_ty; typedef struct token_ty token_ty; struct token_ty { token_type_ty type; char *string; /* for token_type_string, token_type_symbol */ refcounted_string_list_ty *comment; /* for token_type_string */ int line_number; }; /* There are two different input syntaxes for strings, "abc" and r"abc", and two different input syntaxes for Unicode strings, u"abc" and ur"abc". Which escape sequences are understood, i.e. what is interpreted specially after backslash? "abc" \ \\ \' \" \a\b\f\n\r\t\v \ooo \xnn r"abc" u"abc" \ \\ \' \" \a\b\f\n\r\t\v \ooo \xnn \unnnn \Unnnnnnnn \N{...} ur"abc" \unnnn The \unnnn values are UTF-16 values; a single \Unnnnnnnn can expand to two \unnnn items. The \ooo and \xnn values are in the current source encoding for byte strings, and Unicode code points for Unicode strings. */ static int phase7_getuc (int quote_char, bool triple, bool interpret_ansic, bool interpret_unicode, unsigned int *backslash_counter) { int c; for (;;) { /* Use phase 2, because phase 3 elides comments. */ c = phase2_getc (); if (c == UEOF) return P7_EOF; if (c == quote_char && (interpret_ansic || (*backslash_counter & 1) == 0)) { if (triple) { int c1 = phase2_getc (); if (c1 == quote_char) { int c2 = phase2_getc (); if (c2 == quote_char) return P7_STRING_END; phase2_ungetc (c2); } phase2_ungetc (c1); return UNICODE (c); } else return P7_STRING_END; } if (c == '\n') { if (triple) { *backslash_counter = 0; return UNICODE ('\n'); } /* In r"..." and ur"..." strings, newline is only allowed immediately after an odd number of backslashes (although the backslashes are not interpreted!). */ if (!(interpret_ansic || (*backslash_counter & 1) == 0)) { *backslash_counter = 0; return UNICODE ('\n'); } phase2_ungetc (c); error_with_progname = false; error (0, 0, _("%s:%d: warning: unterminated string"), logical_file_name, line_number); error_with_progname = true; return P7_STRING_END; } if (c != '\\') { *backslash_counter = 0; return UNICODE (c); } /* Backslash handling. */ if (!interpret_ansic && !interpret_unicode) { ++*backslash_counter; return UNICODE ('\\'); } /* Dispatch according to the character following the backslash. */ c = phase2_getc (); if (c == UEOF) { ++*backslash_counter; return UNICODE ('\\'); } if (interpret_ansic) switch (c) { case '\n': continue; case '\\': ++*backslash_counter; return UNICODE (c); case '\'': case '"': *backslash_counter = 0; return UNICODE (c); case 'a': *backslash_counter = 0; return UNICODE ('\a'); case 'b': *backslash_counter = 0; return UNICODE ('\b'); case 'f': *backslash_counter = 0; return UNICODE ('\f'); case 'n': *backslash_counter = 0; return UNICODE ('\n'); case 'r': *backslash_counter = 0; return UNICODE ('\r'); case 't': *backslash_counter = 0; return UNICODE ('\t'); case 'v': *backslash_counter = 0; return UNICODE ('\v'); case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': { int n = c - '0'; c = phase2_getc (); if (c != UEOF) { if (c >= '0' && c <= '7') { n = (n << 3) + (c - '0'); c = phase2_getc (); if (c != UEOF) { if (c >= '0' && c <= '7') n = (n << 3) + (c - '0'); else phase2_ungetc (c); } } else phase2_ungetc (c); } *backslash_counter = 0; if (interpret_unicode) return UNICODE (n); else return (unsigned char) n; } case 'x': { int c1 = phase2_getc (); int n1; if (c1 >= '0' && c1 <= '9') n1 = c1 - '0'; else if (c1 >= 'A' && c1 <= 'F') n1 = c1 - 'A' + 10; else if (c1 >= 'a' && c1 <= 'f') n1 = c1 - 'a' + 10; else n1 = -1; if (n1 >= 0) { int c2 = phase2_getc (); int n2; if (c2 >= '0' && c2 <= '9') n2 = c2 - '0'; else if (c2 >= 'A' && c2 <= 'F') n2 = c2 - 'A' + 10; else if (c2 >= 'a' && c2 <= 'f') n2 = c2 - 'a' + 10; else n2 = -1; if (n2 >= 0) { int n = (n1 << 4) + n2; *backslash_counter = 0; if (interpret_unicode) return UNICODE (n); else return (unsigned char) n; } phase2_ungetc (c2); } phase2_ungetc (c1); phase2_ungetc (c); ++*backslash_counter; return UNICODE ('\\'); } } if (interpret_unicode) { if (c == 'u') { unsigned char buf[4]; unsigned int n = 0; int i; for (i = 0; i < 4; i++) { int c1 = phase2_getc (); if (c1 >= '0' && c1 <= '9') n = (n << 4) + (c1 - '0'); else if (c1 >= 'A' && c1 <= 'F') n = (n << 4) + (c1 - 'A' + 10); else if (c1 >= 'a' && c1 <= 'f') n = (n << 4) + (c1 - 'a' + 10); else { phase2_ungetc (c1); while (--i >= 0) phase2_ungetc (buf[i]); phase2_ungetc (c); ++*backslash_counter; return UNICODE ('\\'); } buf[i] = c1; } *backslash_counter = 0; return UNICODE (n); } if (interpret_ansic) { if (c == 'U') { unsigned char buf[8]; unsigned int n = 0; int i; for (i = 0; i < 8; i++) { int c1 = phase2_getc (); if (c1 >= '0' && c1 <= '9') n = (n << 4) + (c1 - '0'); else if (c1 >= 'A' && c1 <= 'F') n = (n << 4) + (c1 - 'A' + 10); else if (c1 >= 'a' && c1 <= 'f') n = (n << 4) + (c1 - 'a' + 10); else { phase2_ungetc (c1); while (--i >= 0) phase2_ungetc (buf[i]); phase2_ungetc (c); ++*backslash_counter; return UNICODE ('\\'); } buf[i] = c1; } if (n < 0x110000) { *backslash_counter = 0; return UNICODE (n); } error_with_progname = false; error (0, 0, _("%s:%d: warning: invalid Unicode character"), logical_file_name, line_number); error_with_progname = true; while (--i >= 0) phase2_ungetc (buf[i]); phase2_ungetc (c); ++*backslash_counter; return UNICODE ('\\'); } if (c == 'N') { int c1 = phase2_getc (); if (c1 == '{') { unsigned char buf[UNINAME_MAX + 1]; int i; unsigned int n; for (i = 0; i < UNINAME_MAX; i++) { int c2 = phase2_getc (); if (!(c2 >= ' ' && c2 <= '~')) { phase2_ungetc (c2); while (--i >= 0) phase2_ungetc (buf[i]); phase2_ungetc (c1); phase2_ungetc (c); ++*backslash_counter; return UNICODE ('\\'); } if (c2 == '}') break; buf[i] = c2; } buf[i] = '\0'; n = unicode_name_character ((char *) buf); if (n != UNINAME_INVALID) { *backslash_counter = 0; return UNICODE (n); } phase2_ungetc ('}'); while (--i >= 0) phase2_ungetc (buf[i]); } phase2_ungetc (c1); phase2_ungetc (c); ++*backslash_counter; return UNICODE ('\\'); } } } phase2_ungetc (c); ++*backslash_counter; return UNICODE ('\\'); } } /* Combine characters into tokens. Discard whitespace except newlines at the end of logical lines. */ /* Number of pending open parentheses/braces/brackets. */ static int open_pbb; static token_ty phase5_pushback[1]; static int phase5_pushback_length; static void phase5_get (token_ty *tp) { int c; if (phase5_pushback_length) { *tp = phase5_pushback[--phase5_pushback_length]; return; } for (;;) { tp->line_number = line_number; c = phase3_getc (); switch (c) { case UEOF: tp->type = token_type_eof; return; case ' ': case '\t': case '\f': /* Ignore whitespace and comments. */ continue; case '\n': if (last_non_comment_line > last_comment_line) savable_comment_reset (); /* Ignore newline if and only if it is used for implicit line joining. */ if (open_pbb > 0) continue; tp->type = token_type_other; return; } last_non_comment_line = tp->line_number; switch (c) { case '.': { int c1 = phase3_getc (); phase3_ungetc (c1); if (!(c1 >= '0' && c1 <= '9')) { tp->type = token_type_other; return; } } /* FALLTHROUGH */ case 'A': case 'B': case 'C': case 'D': case 'E': case 'F': case 'G': case 'H': case 'I': case 'J': case 'K': case 'L': case 'M': case 'N': case 'O': case 'P': case 'Q': case 'S': case 'T': case 'V': case 'W': case 'X': case 'Y': case 'Z': case '_': case 'a': case 'b': case 'c': case 'd': case 'e': case 'f': case 'g': case 'h': case 'i': case 'j': case 'k': case 'l': case 'm': case 'n': case 'o': case 'p': case 'q': case 's': case 't': case 'v': case 'w': case 'x': case 'y': case 'z': case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': symbol: /* Symbol, or part of a number. */ { static char *buffer; static int bufmax; int bufpos; bufpos = 0; for (;;) { if (bufpos >= bufmax) { bufmax = 2 * bufmax + 10; buffer = xrealloc (buffer, bufmax); } buffer[bufpos++] = c; c = phase3_getc (); switch (c) { case 'A': case 'B': case 'C': case 'D': case 'E': case 'F': case 'G': case 'H': case 'I': case 'J': case 'K': case 'L': case 'M': case 'N': case 'O': case 'P': case 'Q': case 'R': case 'S': case 'T': case 'U': case 'V': case 'W': case 'X': case 'Y': case 'Z': case '_': case 'a': case 'b': case 'c': case 'd': case 'e': case 'f': case 'g': case 'h': case 'i': case 'j': case 'k': case 'l': case 'm': case 'n': case 'o': case 'p': case 'q': case 'r': case 's': case 't': case 'u': case 'v': case 'w': case 'x': case 'y': case 'z': case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': continue; default: phase3_ungetc (c); break; } break; } if (bufpos >= bufmax) { bufmax = 2 * bufmax + 10; buffer = xrealloc (buffer, bufmax); } buffer[bufpos] = '\0'; tp->string = xstrdup (buffer); tp->type = token_type_symbol; return; } /* Strings. */ { struct mixed_string_buffer literal; int quote_char; bool interpret_ansic; bool interpret_unicode; bool triple; unsigned int backslash_counter; case 'R': case 'r': { int c1 = phase2_getc (); if (c1 == '"' || c1 == '\'') { quote_char = c1; interpret_ansic = false; interpret_unicode = false; goto string; } phase2_ungetc (c1); goto symbol; } case 'U': case 'u': { int c1 = phase2_getc (); if (c1 == '"' || c1 == '\'') { quote_char = c1; interpret_ansic = true; interpret_unicode = true; goto string; } if (c1 == 'R' || c1 == 'r') { int c2 = phase2_getc (); if (c2 == '"' || c2 == '\'') { quote_char = c2; interpret_ansic = false; interpret_unicode = true; goto string; } phase2_ungetc (c2); } phase2_ungetc (c1); goto symbol; } case '"': case '\'': quote_char = c; interpret_ansic = true; interpret_unicode = false; string: triple = false; lexical_context = lc_string; { int c1 = phase2_getc (); if (c1 == quote_char) { int c2 = phase2_getc (); if (c2 == quote_char) triple = true; else { phase2_ungetc (c2); phase2_ungetc (c1); } } else phase2_ungetc (c1); } backslash_counter = 0; /* Start accumulating the string. */ init_mixed_string_buffer (&literal, lc_string); for (;;) { int uc = phase7_getuc (quote_char, triple, interpret_ansic, interpret_unicode, &backslash_counter); if (uc == P7_EOF || uc == P7_STRING_END) break; if (IS_UNICODE (uc)) assert (UNICODE_VALUE (uc) >= 0 && UNICODE_VALUE (uc) < 0x110000); mixed_string_buffer_append (&literal, uc); } tp->string = xstrdup (mixed_string_buffer_result (&literal)); free_mixed_string_buffer (&literal); tp->comment = add_reference (savable_comment); lexical_context = lc_outside; tp->type = token_type_string; return; } case '(': open_pbb++; tp->type = token_type_lparen; return; case ')': if (open_pbb > 0) open_pbb--; tp->type = token_type_rparen; return; case ',': tp->type = token_type_comma; return; case '[': case '{': open_pbb++; tp->type = (c == '[' ? token_type_lbracket : token_type_other); return; case ']': case '}': if (open_pbb > 0) open_pbb--; tp->type = (c == ']' ? token_type_rbracket : token_type_other); return; default: /* We could carefully recognize each of the 2 and 3 character operators, but it is not necessary, as we only need to recognize gettext invocations. Don't bother. */ tp->type = token_type_other; return; } } } /* Supports only one pushback token. */ static void phase5_unget (token_ty *tp) { if (tp->type != token_type_eof) { if (phase5_pushback_length == SIZEOF (phase5_pushback)) abort (); phase5_pushback[phase5_pushback_length++] = *tp; } } /* Combine adjacent strings to form a single string. Note that the end of a logical line appears as a token of its own, therefore strings that belong to different logical lines will not be concatenated. */ static void x_python_lex (token_ty *tp) { phase5_get (tp); if (tp->type != token_type_string) return; for (;;) { token_ty tmp; size_t len; phase5_get (&tmp); if (tmp.type != token_type_string) { phase5_unget (&tmp); return; } len = strlen (tp->string); tp->string = xrealloc (tp->string, len + strlen (tmp.string) + 1); strcpy (tp->string + len, tmp.string); free (tmp.string); } } /* ========================= Extracting strings. ========================== */ /* Context lookup table. */ static flag_context_list_table_ty *flag_context_list_table; /* The file is broken into tokens. Scan the token stream, looking for a keyword, followed by a left paren, followed by a string. When we see this sequence, we have something to remember. We assume we are looking at a valid C or C++ program, and leave the complaints about the grammar to the compiler. Normal handling: Look for keyword ( ... msgid ... ) Plural handling: Look for keyword ( ... msgid ... msgid_plural ... ) We use recursion because the arguments before msgid or between msgid and msgid_plural can contain subexpressions of the same form. */ /* Extract messages until the next balanced closing parenthesis or bracket. Extracted messages are added to MLP. DELIM can be either token_type_rparen or token_type_rbracket, or token_type_eof to accept both. Return true upon eof, false upon closing parenthesis or bracket. */ static bool extract_balanced (message_list_ty *mlp, token_type_ty delim, flag_context_ty outer_context, flag_context_list_iterator_ty context_iter, struct arglist_parser *argparser) { /* Current argument number. */ int arg = 1; /* 0 when no keyword has been seen. 1 right after a keyword is seen. */ int state; /* Parameters of the keyword just seen. Defined only in state 1. */ const struct callshapes *next_shapes = NULL; /* Context iterator that will be used if the next token is a '('. */ flag_context_list_iterator_ty next_context_iter = passthrough_context_list_iterator; /* Current context. */ flag_context_ty inner_context = inherited_context (outer_context, flag_context_list_iterator_advance (&context_iter)); /* Start state is 0. */ state = 0; for (;;) { token_ty token; x_python_lex (&token); switch (token.type) { case token_type_symbol: { void *keyword_value; if (hash_find_entry (&keywords, token.string, strlen (token.string), &keyword_value) == 0) { next_shapes = (const struct callshapes *) keyword_value; state = 1; } else state = 0; } next_context_iter = flag_context_list_iterator ( flag_context_list_table_lookup ( flag_context_list_table, token.string, strlen (token.string))); free (token.string); continue; case token_type_lparen: if (extract_balanced (mlp, token_type_rparen, inner_context, next_context_iter, arglist_parser_alloc (mlp, state ? next_shapes : NULL))) { xgettext_current_source_encoding = po_charset_utf8; arglist_parser_done (argparser, arg); xgettext_current_source_encoding = xgettext_current_file_source_encoding; return true; } next_context_iter = null_context_list_iterator; state = 0; continue; case token_type_rparen: if (delim == token_type_rparen || delim == token_type_eof) { xgettext_current_source_encoding = po_charset_utf8; arglist_parser_done (argparser, arg); xgettext_current_source_encoding = xgettext_current_file_source_encoding; return false; } next_context_iter = null_context_list_iterator; state = 0; continue; case token_type_comma: arg++; inner_context = inherited_context (outer_context, flag_context_list_iterator_advance ( &context_iter)); next_context_iter = passthrough_context_list_iterator; state = 0; continue; case token_type_lbracket: if (extract_balanced (mlp, token_type_rbracket, null_context, null_context_list_iterator, arglist_parser_alloc (mlp, NULL))) { xgettext_current_source_encoding = po_charset_utf8; arglist_parser_done (argparser, arg); xgettext_current_source_encoding = xgettext_current_file_source_encoding; return true; } next_context_iter = null_context_list_iterator; state = 0; continue; case token_type_rbracket: if (delim == token_type_rbracket || delim == token_type_eof) { xgettext_current_source_encoding = po_charset_utf8; arglist_parser_done (argparser, arg); xgettext_current_source_encoding = xgettext_current_file_source_encoding; return false; } next_context_iter = null_context_list_iterator; state = 0; continue; case token_type_string: { lex_pos_ty pos; pos.file_name = logical_file_name; pos.line_number = token.line_number; xgettext_current_source_encoding = po_charset_utf8; if (extract_all) remember_a_message (mlp, NULL, token.string, inner_context, &pos, NULL, token.comment); else arglist_parser_remember (argparser, arg, token.string, inner_context, pos.file_name, pos.line_number, token.comment); xgettext_current_source_encoding = xgettext_current_file_source_encoding; } drop_reference (token.comment); next_context_iter = null_context_list_iterator; state = 0; continue; case token_type_eof: xgettext_current_source_encoding = po_charset_utf8; arglist_parser_done (argparser, arg); xgettext_current_source_encoding = xgettext_current_file_source_encoding; return true; case token_type_other: next_context_iter = null_context_list_iterator; state = 0; continue; default: abort (); } } } void extract_python (FILE *f, const char *real_filename, const char *logical_filename, flag_context_list_table_ty *flag_table, msgdomain_list_ty *mdlp) { message_list_ty *mlp = mdlp->item[0]->messages; fp = f; real_file_name = real_filename; logical_file_name = xstrdup (logical_filename); line_number = 1; lexical_context = lc_outside; last_comment_line = -1; last_non_comment_line = -1; xgettext_current_file_source_encoding = xgettext_global_source_encoding; #if HAVE_ICONV xgettext_current_file_source_iconv = xgettext_global_source_iconv; #endif xgettext_current_source_encoding = xgettext_current_file_source_encoding; #if HAVE_ICONV xgettext_current_source_iconv = xgettext_current_file_source_iconv; #endif continuation_or_nonblank_line = false; open_pbb = 0; flag_context_list_table = flag_table; init_keywords (); /* Eat tokens until eof is seen. When extract_balanced returns due to an unbalanced closing parenthesis, just restart it. */ while (!extract_balanced (mlp, token_type_eof, null_context, null_context_list_iterator, arglist_parser_alloc (mlp, NULL))) ; fp = NULL; real_file_name = NULL; logical_file_name = NULL; line_number = 0; }