xref: /src/external/gpl2/gettext/dist/gettext-tools/src/x-python.c

/* xgettext Python backend.
   Copyright (C) 2002-2003, 2005-2006 Free Software Foundation, Inc.

   This file was written by Bruno Haible <haible (at) clisp.cons.org>, 2002.

   This program is free software; you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by
   the Free Software Foundation; either version 2, or (at your option)
   any later version.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU General Public License for more details.

   You should have received a copy of the GNU General Public License
   along with this program; if not, write to the Free Software Foundation,
   Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.  */

#ifdef HAVE_CONFIG_H
# include "config.h"
#endif

#include <assert.h>
#include <errno.h>
#include <stdbool.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>

#include "message.h"
#include "xgettext.h"
#include "x-python.h"
#include "error.h"
#include "error-progname.h"
#include "progname.h"
#include "basename.h"
#include "xerror.h"
#include "xvasprintf.h"
#include "xalloc.h"
#include "exit.h"
#include "c-strstr.h"
#include "c-ctype.h"
#include "po-charset.h"
#include "uniname.h"
#include "utf16-ucs4.h"
#include "utf8-ucs4.h"
#include "ucs4-utf8.h"
#include "gettext.h"

#define _(s) gettext(s)

#define max(a,b) ((a) > (b) ? (a) : (b))

#define SIZEOF(a) (sizeof(a) / sizeof(a[0]))


/* The Python syntax is defined in the Python Reference Manual
   /usr/share/doc/packages/python/html/ref/index.html.
   See also Python-2.0/Parser/tokenizer.c, Python-2.0/Python/compile.c,
   Python-2.0/Objects/unicodeobject.c.  */


/* ====================== Keyword set customization.  ====================== */

/* If true extract all strings.  */
static bool extract_all = false;

static hash_table keywords;
static bool default_keywords = true;


void
x_python_extract_all ()
{
  extract_all = true;
}


void
x_python_keyword (const char *name)
{
  if (name == NULL)
    default_keywords = false;
  else
    {
      const char *end;
      struct callshape shape;
      const char *colon;

      if (keywords.table == NULL)
	hash_init (&keywords, 100);

      split_keywordspec (name, &end, &shape);

      /* The characters between name and end should form a valid C identifier.
	 A colon means an invalid parse in split_keywordspec().  */
      colon = strchr (name, ':');
      if (colon == NULL || colon >= end)
	insert_keyword_callshape (&keywords, name, end - name, &shape);
    }
}

/* Finish initializing the keywords hash table.
   Called after argument processing, before each file is processed.  */
static void
init_keywords ()
{
  if (default_keywords)
    {
      /* When adding new keywords here, also update the documentation in
	 xgettext.texi!  */
      x_python_keyword ("gettext");
      x_python_keyword ("ugettext");
      x_python_keyword ("dgettext:2");
      x_python_keyword ("ngettext:1,2");
      x_python_keyword ("ungettext:1,2");
      x_python_keyword ("dngettext:2,3");
      x_python_keyword ("_");
      default_keywords = false;
    }
}

void
init_flag_table_python ()
{
  xgettext_record_flag ("gettext:1:pass-python-format");
  xgettext_record_flag ("ugettext:1:pass-python-format");
  xgettext_record_flag ("dgettext:2:pass-python-format");
  xgettext_record_flag ("ngettext:1:pass-python-format");
  xgettext_record_flag ("ngettext:2:pass-python-format");
  xgettext_record_flag ("ungettext:1:pass-python-format");
  xgettext_record_flag ("ungettext:2:pass-python-format");
  xgettext_record_flag ("dngettext:2:pass-python-format");
  xgettext_record_flag ("dngettext:3:pass-python-format");
  xgettext_record_flag ("_:1:pass-python-format");
  /* xgettext_record_flag ("%:1:python-format"); // % is an infix operator! */
}


/* ======================== Reading of characters.  ======================== */

/* Real filename, used in error messages about the input file.  */
static const char *real_file_name;

/* Logical filename and line number, used to label the extracted messages.  */
static char *logical_file_name;
static int line_number;

/* The input file stream.  */
static FILE *fp;


/* 1. line_number handling.  */

/* Maximum used, roughly a safer MB_LEN_MAX.  */
#define MAX_PHASE1_PUSHBACK 16
static unsigned char phase1_pushback[MAX_PHASE1_PUSHBACK];
static int phase1_pushback_length;

/* Read the next single byte from the input file.  */
static int
phase1_getc ()
{
  int c;

  if (phase1_pushback_length)
    c = phase1_pushback[--phase1_pushback_length];
  else
    {
      c = getc (fp);

      if (c == EOF)
	{
	  if (ferror (fp))
	    error (EXIT_FAILURE, errno, _("error while reading \"%s\""),
		   real_file_name);
	  return EOF;
	}
    }

  if (c == '\n')
    ++line_number;

  return c;
}

/* Supports MAX_PHASE1_PUSHBACK characters of pushback.  */
static void
phase1_ungetc (int c)
{
  if (c != EOF)
    {
      if (c == '\n')
	--line_number;

      if (phase1_pushback_length == SIZEOF (phase1_pushback))
	abort ();
      phase1_pushback[phase1_pushback_length++] = c;
    }
}


/* Phase 2: Conversion to Unicode.
   This is done early because PEP 0263 specifies that conversion to Unicode
   conceptually occurs before tokenization.  A test case where it matters
   is with encodings like BIG5: when a double-byte character ending in 0x5C
   is followed by '\' or 'u0021', the tokenizer must not treat the second
   half of the double-byte character as a backslash.  */

/* End-of-file indicator for functions returning an UCS-4 character.  */
#define UEOF -1

static int phase2_pushback[max (9, UNINAME_MAX + 3)];
static int phase2_pushback_length;

/* Read the next Unicode UCS-4 character from the input file.  */
static int
phase2_getc ()
{
  if (phase2_pushback_length)
    return phase2_pushback[--phase2_pushback_length];

  if (xgettext_current_source_encoding == po_charset_ascii)
    {
      int c = phase1_getc ();
      if (c == EOF)
	return UEOF;
      if (!c_isascii (c))
	{
	  char buffer[21];
	  sprintf (buffer, ":%ld", (long) line_number);
	  multiline_error (xstrdup (""),
			   xasprintf (_("\
Non-ASCII string at %s%s.\n\
Please specify the source encoding through --from-code or through a comment\n\
as specified in http://www.python.org/peps/pep-0263.html.\n"),
			   real_file_name, buffer));
	  exit (EXIT_FAILURE);
	}
      return c;
    }
  else if (xgettext_current_source_encoding != po_charset_utf8)
    {
#if HAVE_ICONV
      /* Use iconv on an increasing number of bytes.  Read only as many bytes
	 through phase1_getc as needed.  This is needed to give reasonable
	 interactive behaviour when fp is connected to an interactive tty.  */
      unsigned char buf[MAX_PHASE1_PUSHBACK];
      size_t bufcount;
      int c = phase1_getc ();
      if (c == EOF)
	return UEOF;
      buf[0] = (unsigned char) c;
      bufcount = 1;

      for (;;)
	{
	  unsigned char scratchbuf[6];
	  const char *inptr = (const char *) &buf[0];
	  size_t insize = bufcount;
	  char *outptr = (char *) &scratchbuf[0];
	  size_t outsize = sizeof (scratchbuf);

	  size_t res = iconv (xgettext_current_source_iconv,
			      (ICONV_CONST char **) &inptr, &insize,
			      &outptr, &outsize);
	  /* We expect that a character has been produced if and only if
	     some input bytes have been consumed.  */
	  if ((insize < bufcount) != (outsize < sizeof (scratchbuf)))
	    abort ();
	  if (outsize == sizeof (scratchbuf))
	    {
	      /* No character has been produced.  Must be an error.  */
	      if (res != (size_t)(-1))
		abort ();

	      if (errno == EILSEQ)
		{
		  /* An invalid multibyte sequence was encountered.  */
		  multiline_error (xstrdup (""),
				   xasprintf (_("\
%s:%d: Invalid multibyte sequence.\n\
Please specify the correct source encoding through --from-code or through a\n\
comment as specified in http://www.python.org/peps/pep-0263.html.\n"),
				   real_file_name, line_number));
		  exit (EXIT_FAILURE);
		}
	      else if (errno == EINVAL)
		{
		  /* An incomplete multibyte character.  */
		  int c;

		  if (bufcount == MAX_PHASE1_PUSHBACK)
		    {
		      /* An overlong incomplete multibyte sequence was
			 encountered.  */
		      multiline_error (xstrdup (""),
				       xasprintf (_("\
%s:%d: Long incomplete multibyte sequence.\n\
Please specify the correct source encoding through --from-code or through a\n\
comment as specified in http://www.python.org/peps/pep-0263.html.\n"),
				       real_file_name, line_number));
		      exit (EXIT_FAILURE);
		    }

		  /* Read one more byte and retry iconv.  */
		  c = phase1_getc ();
		  if (c == EOF)
		    {
		      multiline_error (xstrdup (""),
				       xasprintf (_("\
%s:%d: Incomplete multibyte sequence at end of file.\n\
Please specify the correct source encoding through --from-code or through a\n\
comment as specified in http://www.python.org/peps/pep-0263.html.\n"),
				       real_file_name, line_number));
		      exit (EXIT_FAILURE);
		    }
		  if (c == '\n')
		    {
		      multiline_error (xstrdup (""),
				       xasprintf (_("\
%s:%d: Incomplete multibyte sequence at end of line.\n\
Please specify the correct source encoding through --from-code or through a\n\
comment as specified in http://www.python.org/peps/pep-0263.html.\n"),
				       real_file_name, line_number - 1));
		      exit (EXIT_FAILURE);
		    }
		  buf[bufcount++] = (unsigned char) c;
		}
	      else
		error (EXIT_FAILURE, errno, _("%s:%d: iconv failure"),
		       real_file_name, line_number);
	    }
	  else
	    {
	      size_t outbytes = sizeof (scratchbuf) - outsize;
	      size_t bytes = bufcount - insize;
	      unsigned int uc;

	      /* We expect that one character has been produced.  */
	      if (bytes == 0)
		abort ();
	      if (outbytes == 0)
		abort ();
	      /* Push back the unused bytes.  */
	      while (insize > 0)
		phase1_ungetc (buf[--insize]);
	      /* Convert the character from UTF-8 to UCS-4.  */
	      if (u8_mbtouc (&uc, scratchbuf, outbytes) < outbytes)
		{
		  /* scratchbuf contains an out-of-range Unicode character
		     (> 0x10ffff).  */
		  multiline_error (xstrdup (""),
				   xasprintf (_("\
%s:%d: Invalid multibyte sequence.\n\
Please specify the source encoding through --from-code or through a comment\n\
as specified in http://www.python.org/peps/pep-0263.html.\n"),
				   real_file_name, line_number));
		  exit (EXIT_FAILURE);
		}
	      return uc;
	    }
	}
#else
      /* If we don't have iconv(), the only supported values for
	 xgettext_global_source_encoding and thus also for
	 xgettext_current_source_encoding are ASCII and UTF-8.  */
      abort ();
#endif
    }
  else
    {
      /* Read an UTF-8 encoded character.  */
      unsigned char buf[6];
      unsigned int count;
      int c;
      unsigned int uc;

      c = phase1_getc ();
      if (c == EOF)
	return UEOF;
      buf[0] = c;
      count = 1;

      if (buf[0] >= 0xc0)
	{
	  c = phase1_getc ();
	  if (c == EOF)
	    return UEOF;
	  buf[1] = c;
	  count = 2;
	}

      if (buf[0] >= 0xe0
	  && ((buf[1] ^ 0x80) < 0x40))
	{
	  c = phase1_getc ();
	  if (c == EOF)
	    return UEOF;
	  buf[2] = c;
	  count = 3;
	}

      if (buf[0] >= 0xf0
	  && ((buf[1] ^ 0x80) < 0x40)
	  && ((buf[2] ^ 0x80) < 0x40))
	{
	  c = phase1_getc ();
	  if (c == EOF)
	    return UEOF;
	  buf[3] = c;
	  count = 4;
	}

      if (buf[0] >= 0xf8
	  && ((buf[1] ^ 0x80) < 0x40)
	  && ((buf[2] ^ 0x80) < 0x40)
	  && ((buf[3] ^ 0x80) < 0x40))
	{
	  c = phase1_getc ();
	  if (c == EOF)
	    return UEOF;
	  buf[4] = c;
	  count = 5;
	}

      if (buf[0] >= 0xfc
	  && ((buf[1] ^ 0x80) < 0x40)
	  && ((buf[2] ^ 0x80) < 0x40)
	  && ((buf[3] ^ 0x80) < 0x40)
	  && ((buf[4] ^ 0x80) < 0x40))
	{
	  c = phase1_getc ();
	  if (c == EOF)
	    return UEOF;
	  buf[5] = c;
	  count = 6;
	}

      u8_mbtouc (&uc, buf, count);
      return uc;
    }
}

/* Supports max (9, UNINAME_MAX + 3) pushback characters.  */
static void
phase2_ungetc (int c)
{
  if (c != UEOF)
    {
      if (phase2_pushback_length == SIZEOF (phase2_pushback))
	abort ();
      phase2_pushback[phase2_pushback_length++] = c;
    }
}


/* ========================= Accumulating strings.  ======================== */

/* A string buffer type that allows appending Unicode characters.
   Returns the entire string in UTF-8 encoding.  */

struct unicode_string_buffer
{
  /* The part of the string that has already been converted to UTF-8.  */
  char *utf8_buffer;
  size_t utf8_buflen;
  size_t utf8_allocated;
};

/* Initialize a 'struct unicode_string_buffer' to empty.  */
static inline void
init_unicode_string_buffer (struct unicode_string_buffer *bp)
{
  bp->utf8_buffer = NULL;
  bp->utf8_buflen = 0;
  bp->utf8_allocated = 0;
}

/* Auxiliary function: Ensure count more bytes are available in bp->utf8.  */
static inline void
unicode_string_buffer_append_unicode_grow (struct unicode_string_buffer *bp,
					   size_t count)
{
  if (bp->utf8_buflen + count > bp->utf8_allocated)
    {
      size_t new_allocated = 2 * bp->utf8_allocated + 10;
      if (new_allocated < bp->utf8_buflen + count)
	new_allocated = bp->utf8_buflen + count;
      bp->utf8_allocated = new_allocated;
      bp->utf8_buffer = xrealloc (bp->utf8_buffer, new_allocated);
    }
}

/* Auxiliary function: Append a Unicode character to bp->utf8.
   uc must be < 0x110000.  */
static inline void
unicode_string_buffer_append_unicode (struct unicode_string_buffer *bp,
				      unsigned int uc)
{
  unsigned char utf8buf[6];
  int count = u8_uctomb (utf8buf, uc, 6);

  if (count < 0)
    /* The caller should have ensured that uc is not out-of-range.  */
    abort ();

  unicode_string_buffer_append_unicode_grow (bp, count);
  memcpy (bp->utf8_buffer + bp->utf8_buflen, utf8buf, count);
  bp->utf8_buflen += count;
}

/* Return the string buffer's contents.  */
static char *
unicode_string_buffer_result (struct unicode_string_buffer *bp)
{
  /* NUL-terminate it.  */
  unicode_string_buffer_append_unicode_grow (bp, 1);
  bp->utf8_buffer[bp->utf8_buflen] = '\0';
  /* Return it.  */
  return bp->utf8_buffer;
}

/* Free the memory pointed to by a 'struct unicode_string_buffer'.  */
static inline void
free_unicode_string_buffer (struct unicode_string_buffer *bp)
{
  free (bp->utf8_buffer);
}


/* ======================== Accumulating comments.  ======================== */


/* Accumulating a single comment line.  */

static struct unicode_string_buffer comment_buffer;

static inline void
comment_start ()
{
  comment_buffer.utf8_buflen = 0;
}

static inline bool
comment_at_start ()
{
  return (comment_buffer.utf8_buflen == 0);
}

static inline void
comment_add (int c)
{
  unicode_string_buffer_append_unicode (&comment_buffer, c);
}

static inline const char *
comment_line_end ()
{
  char *buffer = unicode_string_buffer_result (&comment_buffer);
  size_t buflen = strlen (buffer);

  while (buflen >= 1
	 && (buffer[buflen - 1] == ' ' || buffer[buflen - 1] == '\t'))
    --buflen;
  buffer[buflen] = '\0';
  savable_comment_add (buffer);
  return buffer;
}


/* These are for tracking whether comments count as immediately before
   keyword.  */
static int last_comment_line;
static int last_non_comment_line;


/* ======================== Recognizing comments.  ======================== */


/* Recognizing the "coding" comment.
   As specified in PEP 0263, it takes the form
     "coding" [":"|"="] {alphanumeric or "-" or "_" or "*"}*
   and is located in a comment in a line that
     - is either the first or second line,
     - is not a continuation line,
     - contains no other tokens except this comment.  */

/* Canonicalized encoding name for the current input file.  */
static const char *xgettext_current_file_source_encoding;

#if HAVE_ICONV
/* Converter from xgettext_current_file_source_encoding to UTF-8 (except from
   ASCII or UTF-8, when this conversion is a no-op).  */
static iconv_t xgettext_current_file_source_iconv;
#endif

static inline void
set_current_file_source_encoding (const char *canon_encoding)
{
  xgettext_current_file_source_encoding = canon_encoding;

  if (xgettext_current_file_source_encoding != po_charset_ascii
      && xgettext_current_file_source_encoding != po_charset_utf8)
    {
#if HAVE_ICONV
      iconv_t cd;

      /* Avoid glibc-2.1 bug with EUC-KR.  */
# if (__GLIBC__ - 0 == 2 && __GLIBC_MINOR__ - 0 <= 1) && !defined _LIBICONV_VERSION
      if (strcmp (xgettext_current_file_source_encoding, "EUC-KR") == 0)
	cd = (iconv_t)(-1);
      else
# endif
      cd = iconv_open (po_charset_utf8, xgettext_current_file_source_encoding);
      if (cd == (iconv_t)(-1))
	error_at_line (EXIT_FAILURE, 0, logical_file_name, line_number - 1, _("\
Cannot convert from \"%s\" to \"%s\". %s relies on iconv(), \
and iconv() does not support this conversion."),
	       xgettext_current_file_source_encoding, po_charset_utf8,
	       basename (program_name));
      xgettext_current_file_source_iconv = cd;
#else
      error_at_line (EXIT_FAILURE, 0, logical_file_name, line_number - 1, _("\
Cannot convert from \"%s\" to \"%s\". %s relies on iconv(). \
This version was built without iconv()."),
	     xgettext_global_source_encoding, po_charset_utf8,
	     basename (program_name));
#endif
    }

  xgettext_current_source_encoding = xgettext_current_file_source_encoding;
#if HAVE_ICONV
  xgettext_current_source_iconv = xgettext_current_file_source_iconv;
#endif
}

static inline void
try_to_extract_coding (const char *comment)
{
  const char *p = c_strstr (comment, "coding");

  if (p != NULL)
    {
      p += 6;
      if (*p == ':' || *p == '=')
	{
	  p++;
	  while (*p == ' ' || *p == '\t')
	    p++;
	  {
	    const char *encoding_start = p;

	    while (c_isalnum (*p) || *p == '-' || *p == '_' || *p == '.')
	      p++;
	    {
	      const char *encoding_end = p;

	      if (encoding_end > encoding_start)
		{
		  /* Extract the encoding string.  */
		  size_t encoding_len = encoding_end - encoding_start;
		  char *encoding = (char *) xmalloc (encoding_len + 1);

		  memcpy (encoding, encoding_start, encoding_len);
		  encoding[encoding_len] = '\0';

		  {
		    /* Canonicalize it.  */
		    const char *canon_encoding = po_charset_canonicalize (encoding);
		    if (canon_encoding == NULL)
		      {
			error_at_line (0, 0,
				       logical_file_name, line_number - 1, _("\
Unknown encoding \"%s\". Proceeding with ASCII instead."),
				       encoding);
		        canon_encoding = po_charset_ascii;
		      }

		    /* Activate it.  */
		    set_current_file_source_encoding (canon_encoding);
		  }

		  free (encoding);
		}
	    }
	  }
	}
    }
}

/* Tracking whether the current line is a continuation line or contains a
   non-blank character.  */
static bool continuation_or_nonblank_line = false;


/* Phase 3: Outside strings, replace backslash-newline with nothing and a
   comment with nothing.  */

static int
phase3_getc ()
{
  int c;

  for (;;)
    {
      c = phase2_getc ();
      if (c == '\\')
	{
	  c = phase2_getc ();
	  if (c != '\n')
	    {
	      phase2_ungetc (c);
	      /* This shouldn't happen usually, because "A backslash is
		 illegal elsewhere on a line outside a string literal."  */
	      return '\\';
	    }
	  /* Eat backslash-newline.  */
	  continuation_or_nonblank_line = true;
	}
      else if (c == '#')
	{
	  /* Eat a comment.  */
	  const char *comment;

	  last_comment_line = line_number;
	  comment_start ();
	  for (;;)
	    {
	      c = phase2_getc ();
	      if (c == UEOF || c == '\n')
		break;
	      /* We skip all leading white space, but not EOLs.  */
	      if (!(comment_at_start () && (c == ' ' || c == '\t')))
		comment_add (c);
	    }
	  comment = comment_line_end ();
	  if (line_number - 1 <= 2 && !continuation_or_nonblank_line)
	    try_to_extract_coding (comment);
	  continuation_or_nonblank_line = false;
	  return c;
	}
      else
	{
	  if (c == '\n')
	    continuation_or_nonblank_line = false;
	  else if (!(c == ' ' || c == '\t' || c == '\f'))
	    continuation_or_nonblank_line = true;
	  return c;
	}
    }
}

/* Supports only one pushback character.  */
static void
phase3_ungetc (int c)
{
  phase2_ungetc (c);
}


/* ========================= Accumulating strings.  ======================== */

/* Return value of phase7_getuc when EOF is reached.  */
#define P7_EOF (-1)
#define P7_STRING_END (-2)

/* Convert an UTF-16 or UTF-32 code point to a return value that can be
   distinguished from a single-byte return value.  */
#define UNICODE(code) (0x100 + (code))

/* Test a return value of phase7_getuc whether it designates an UTF-16 or
   UTF-32 code point.  */
#define IS_UNICODE(p7_result) ((p7_result) >= 0x100)

/* Extract the UTF-16 or UTF-32 code of a return value that satisfies
   IS_UNICODE.  */
#define UNICODE_VALUE(p7_result) ((p7_result) - 0x100)

/* A string buffer type that allows appending bytes (in the
   xgettext_current_source_encoding) or Unicode characters.
   Returns the entire string in UTF-8 encoding.  */

struct mixed_string_buffer
{
  /* The part of the string that has already been converted to UTF-8.  */
  char *utf8_buffer;
  size_t utf8_buflen;
  size_t utf8_allocated;
  /* The first half of an UTF-16 surrogate character.  */
  unsigned short utf16_surr;
  /* The part of the string that is still in the source encoding.  */
  char *curr_buffer;
  size_t curr_buflen;
  size_t curr_allocated;
};

/* Initialize a 'struct mixed_string_buffer' to empty.  */
static inline void
init_mixed_string_buffer (struct mixed_string_buffer *bp)
{
  bp->utf8_buffer = NULL;
  bp->utf8_buflen = 0;
  bp->utf8_allocated = 0;
  bp->utf16_surr = 0;
  bp->curr_buffer = NULL;
  bp->curr_buflen = 0;
  bp->curr_allocated = 0;
}

/* Auxiliary function: Append a byte to bp->curr.  */
static inline void
mixed_string_buffer_append_byte (struct mixed_string_buffer *bp, unsigned char c)
{
  if (bp->curr_buflen == bp->curr_allocated)
    {
      bp->curr_allocated = 2 * bp->curr_allocated + 10;
      bp->curr_buffer = xrealloc (bp->curr_buffer, bp->curr_allocated);
    }
  bp->curr_buffer[bp->curr_buflen++] = c;
}

/* Auxiliary function: Ensure count more bytes are available in bp->utf8.  */
static inline void
mixed_string_buffer_append_unicode_grow (struct mixed_string_buffer *bp, size_t count)
{
  if (bp->utf8_buflen + count > bp->utf8_allocated)
    {
      size_t new_allocated = 2 * bp->utf8_allocated + 10;
      if (new_allocated < bp->utf8_buflen + count)
	new_allocated = bp->utf8_buflen + count;
      bp->utf8_allocated = new_allocated;
      bp->utf8_buffer = xrealloc (bp->utf8_buffer, new_allocated);
    }
}

/* Auxiliary function: Append a Unicode character to bp->utf8.
   uc must be < 0x110000.  */
static inline void
mixed_string_buffer_append_unicode (struct mixed_string_buffer *bp, unsigned int uc)
{
  unsigned char utf8buf[6];
  int count = u8_uctomb (utf8buf, uc, 6);

  if (count < 0)
    /* The caller should have ensured that uc is not out-of-range.  */
    abort ();

  mixed_string_buffer_append_unicode_grow (bp, count);
  memcpy (bp->utf8_buffer + bp->utf8_buflen, utf8buf, count);
  bp->utf8_buflen += count;
}

/* Auxiliary function: Flush bp->utf16_surr into bp->utf8_buffer.  */
static inline void
mixed_string_buffer_flush_utf16_surr (struct mixed_string_buffer *bp)
{
  if (bp->utf16_surr != 0)
    {
      /* A half surrogate is invalid, therefore use U+FFFD instead.  */
      mixed_string_buffer_append_unicode (bp, 0xfffd);
      bp->utf16_surr = 0;
    }
}

/* Auxiliary function: Flush bp->curr_buffer into bp->utf8_buffer.  */
static inline void
mixed_string_buffer_flush_curr_buffer (struct mixed_string_buffer *bp, int lineno)
{
  if (bp->curr_buflen > 0)
    {
      char *curr;
      size_t count;

      mixed_string_buffer_append_byte (bp, '\0');

      /* Convert from the source encoding to UTF-8.  */
      curr = from_current_source_encoding (bp->curr_buffer,
					   logical_file_name, lineno);

      /* Append it to bp->utf8_buffer.  */
      count = strlen (curr);
      mixed_string_buffer_append_unicode_grow (bp, count);
      memcpy (bp->utf8_buffer + bp->utf8_buflen, curr, count);
      bp->utf8_buflen += count;

      if (curr != bp->curr_buffer)
	free (curr);
      bp->curr_buflen = 0;
    }
}

/* Append a character or Unicode character to a 'struct mixed_string_buffer'.  */
static void
mixed_string_buffer_append (struct mixed_string_buffer *bp, int c)
{
  if (IS_UNICODE (c))
    {
      /* Append a Unicode character.  */

      /* Switch from multibyte character mode to Unicode character mode.  */
      mixed_string_buffer_flush_curr_buffer (bp, line_number);

      /* Test whether this character and the previous one form a Unicode
	 surrogate character pair.  */
      if (bp->utf16_surr != 0
	  && (c >= UNICODE (0xdc00) && c < UNICODE (0xe000)))
	{
	  unsigned short utf16buf[2];
	  unsigned int uc;

	  utf16buf[0] = bp->utf16_surr;
	  utf16buf[1] = UNICODE_VALUE (c);
	  if (u16_mbtouc_aux (&uc, utf16buf, 2) != 2)
	    abort ();

	  mixed_string_buffer_append_unicode (bp, uc);
	  bp->utf16_surr = 0;
	}
      else
	{
	  mixed_string_buffer_flush_utf16_surr (bp);

	  if (c >= UNICODE (0xd800) && c < UNICODE (0xdc00))
	    bp->utf16_surr = UNICODE_VALUE (c);
	  else
	    mixed_string_buffer_append_unicode (bp, UNICODE_VALUE (c));
	}
    }
  else
    {
      /* Append a single byte.  */

      /* Switch from Unicode character mode to multibyte character mode.  */
      mixed_string_buffer_flush_utf16_surr (bp);

      /* When a newline is seen, convert the accumulated multibyte sequence.
	 This ensures a correct line number in the error message in case of
	 a conversion error.  The "- 1" is to account for the newline.  */
      if (c == '\n')
	mixed_string_buffer_flush_curr_buffer (bp, line_number - 1);

      mixed_string_buffer_append_byte (bp, (unsigned char) c);
    }
}

/* Return the string buffer's contents.  */
static char *
mixed_string_buffer_result (struct mixed_string_buffer *bp)
{
  /* Flush all into bp->utf8_buffer.  */
  mixed_string_buffer_flush_utf16_surr (bp);
  mixed_string_buffer_flush_curr_buffer (bp, line_number);
  /* NUL-terminate it.  */
  mixed_string_buffer_append_unicode_grow (bp, 1);
  bp->utf8_buffer[bp->utf8_buflen] = '\0';
  /* Return it.  */
  return bp->utf8_buffer;
}

/* Free the memory pointed to by a 'struct mixed_string_buffer'.  */
static inline void
free_mixed_string_buffer (struct mixed_string_buffer *bp)
{
  free (bp->utf8_buffer);
  free (bp->curr_buffer);
}


/* ========================== Reading of tokens.  ========================== */


enum token_type_ty
{
  token_type_eof,
  token_type_lparen,		/* ( */
  token_type_rparen,		/* ) */
  token_type_comma,		/* , */
  token_type_string,		/* "abc", 'abc', """abc""", '''abc''' */
  token_type_symbol,		/* symbol, number */
  token_type_other		/* misc. operator */
};
typedef enum token_type_ty token_type_ty;

typedef struct token_ty token_ty;
struct token_ty
{
  token_type_ty type;
  char *string;		/* for token_type_string, token_type_symbol */
  refcounted_string_list_ty *comment;	/* for token_type_string */
  int line_number;
};


/* There are two different input syntaxes for strings, "abc" and r"abc",
   and two different input syntaxes for Unicode strings, u"abc" and ur"abc".
   Which escape sequences are understood, i.e. what is interpreted specially
   after backslash?
    "abc"     \<nl> \\ \' \" \a\b\f\n\r\t\v \ooo \xnn
    r"abc"
    u"abc"    \<nl> \\ \' \" \a\b\f\n\r\t\v \ooo \xnn \unnnn \Unnnnnnnn \N{...}
    ur"abc"                                           \unnnn
   The \unnnn values are UTF-16 values; a single \Unnnnnnnn can expand to two
   \unnnn items.  The \ooo and \xnn values are in the current source encoding.
 */

static int
phase7_getuc (int quote_char,
	      bool triple, bool interpret_ansic, bool interpret_unicode,
	      unsigned int *backslash_counter)
{
  int c;

  for (;;)
    {
      /* Use phase 2, because phase 3 elides comments.  */
      c = phase2_getc ();

      if (c == UEOF)
	return P7_EOF;

      if (c == quote_char && (interpret_ansic || (*backslash_counter & 1) == 0))
	{
	  if (triple)
	    {
	      int c1 = phase2_getc ();
	      if (c1 == quote_char)
		{
		  int c2 = phase2_getc ();
		  if (c2 == quote_char)
		    return P7_STRING_END;
		  phase2_ungetc (c2);
		}
	      phase2_ungetc (c1);
	      return UNICODE (c);
	    }
	  else
	    return P7_STRING_END;
	}

      if (c == '\n')
	{
	  if (triple)
	    {
	      *backslash_counter = 0;
	      return UNICODE ('\n');
	    }
	  /* In r"..." and ur"..." strings, newline is only allowed
	     immediately after an odd number of backslashes (although the
	     backslashes are not interpreted!).  */
	  if (!(interpret_ansic || (*backslash_counter & 1) == 0))
	    {
	      *backslash_counter = 0;
	      return UNICODE ('\n');
	    }
	  phase2_ungetc (c);
	  error_with_progname = false;
	  error (0, 0, _("%s:%d: warning: unterminated string"),
		 logical_file_name, line_number);
	  error_with_progname = true;
	  return P7_STRING_END;
	}

      if (c != '\\')
	{
	  *backslash_counter = 0;
	  return UNICODE (c);
	}

      /* Backslash handling.  */

      if (!interpret_ansic && !interpret_unicode)
	{
	  ++*backslash_counter;
	  return UNICODE ('\\');
	}

      /* Dispatch according to the character following the backslash.  */
      c = phase2_getc ();
      if (c == UEOF)
	{
	  ++*backslash_counter;
	  return UNICODE ('\\');
	}

      if (interpret_ansic)
	switch (c)
	  {
	  case '\n':
	    continue;
	  case '\\':
	    ++*backslash_counter;
	    return UNICODE (c);
	  case '\'': case '"':
	    *backslash_counter = 0;
	    return UNICODE (c);
	  case 'a':
	    *backslash_counter = 0;
	    return UNICODE ('\a');
	  case 'b':
	    *backslash_counter = 0;
	    return UNICODE ('\b');
	  case 'f':
	    *backslash_counter = 0;
	    return UNICODE ('\f');
	  case 'n':
	    *backslash_counter = 0;
	    return UNICODE ('\n');
	  case 'r':
	    *backslash_counter = 0;
	    return UNICODE ('\r');
	  case 't':
	    *backslash_counter = 0;
	    return UNICODE ('\t');
	  case 'v':
	    *backslash_counter = 0;
	    return UNICODE ('\v');
	  case '0': case '1': case '2': case '3': case '4':
	  case '5': case '6': case '7':
	    {
	      int n = c - '0';

	      c = phase2_getc ();
	      if (c != UEOF)
		{
		  if (c >= '0' && c <= '7')
		    {
		      n = (n << 3) + (c - '0');
		      c = phase2_getc ();
		      if (c != UEOF)
			{
			  if (c >= '0' && c <= '7')
			    n = (n << 3) + (c - '0');
			  else
			    phase2_ungetc (c);
			}
		    }
		  else
		    phase2_ungetc (c);
		}
	      *backslash_counter = 0;
	      return (unsigned char) n;
	    }
	  case 'x':
	    {
	      int c1 = phase2_getc ();
	      int n1;

	      if (c1 >= '0' && c1 <= '9')
		n1 = c1 - '0';
	      else if (c1 >= 'A' && c1 <= 'F')
		n1 = c1 - 'A' + 10;
	      else if (c1 >= 'a' && c1 <= 'f')
		n1 = c1 - 'a' + 10;
	      else
		n1 = -1;

	      if (n1 >= 0)
		{
		  int c2 = phase2_getc ();
		  int n2;

		  if (c2 >= '0' && c2 <= '9')
		    n2 = c2 - '0';
		  else if (c2 >= 'A' && c2 <= 'F')
		    n2 = c2 - 'A' + 10;
		  else if (c2 >= 'a' && c2 <= 'f')
		    n2 = c2 - 'a' + 10;
		  else
		    n2 = -1;

		  if (n2 >= 0)
		    {
		      *backslash_counter = 0;
		      return (unsigned char) ((n1 << 4) + n2);
		    }

		  phase2_ungetc (c2);
		}
	      phase2_ungetc (c1);
	      phase2_ungetc (c);
	      ++*backslash_counter;
	      return UNICODE ('\\');
	    }
	  }

      if (interpret_unicode)
	{
	  if (c == 'u')
	    {
	      unsigned char buf[4];
	      unsigned int n = 0;
	      int i;

	      for (i = 0; i < 4; i++)
		{
		  int c1 = phase2_getc ();

		  if (c1 >= '0' && c1 <= '9')
		    n = (n << 4) + (c1 - '0');
		  else if (c1 >= 'A' && c1 <= 'F')
		    n = (n << 4) + (c1 - 'A' + 10);
		  else if (c1 >= 'a' && c1 <= 'f')
		    n = (n << 4) + (c1 - 'a' + 10);
		  else
		    {
		      phase2_ungetc (c1);
		      while (--i >= 0)
			phase2_ungetc (buf[i]);
		      phase2_ungetc (c);
		      ++*backslash_counter;
		      return UNICODE ('\\');
		    }

		  buf[i] = c1;
		}
	      *backslash_counter = 0;
	      return UNICODE (n);
	    }

	  if (interpret_ansic)
	    {
	      if (c == 'U')
		{
		  unsigned char buf[8];
		  unsigned int n = 0;
		  int i;

		  for (i = 0; i < 8; i++)
		    {
		      int c1 = phase2_getc ();

		      if (c1 >= '0' && c1 <= '9')
			n = (n << 4) + (c1 - '0');
		      else if (c1 >= 'A' && c1 <= 'F')
			n = (n << 4) + (c1 - 'A' + 10);
		      else if (c1 >= 'a' && c1 <= 'f')
			n = (n << 4) + (c1 - 'a' + 10);
		      else
			{
			  phase2_ungetc (c1);
			  while (--i >= 0)
			    phase2_ungetc (buf[i]);
			  phase2_ungetc (c);
			  ++*backslash_counter;
			  return UNICODE ('\\');
			}

		      buf[i] = c1;
		    }
		  if (n < 0x110000)
		    {
		      *backslash_counter = 0;
		      return UNICODE (n);
		    }

		  error_with_progname = false;
		  error (0, 0, _("%s:%d: warning: invalid Unicode character"),
			 logical_file_name, line_number);
		  error_with_progname = true;

		  while (--i >= 0)
		    phase2_ungetc (buf[i]);
		  phase2_ungetc (c);
		  ++*backslash_counter;
		  return UNICODE ('\\');
		}

	      if (c == 'N')
		{
		  int c1 = phase2_getc ();
		  if (c1 == '{')
		    {
		      unsigned char buf[UNINAME_MAX + 1];
		      int i;
		      unsigned int n;

		      for (i = 0; i < UNINAME_MAX; i++)
			{
			  int c2 = phase2_getc ();
			  if (!(c2 >= ' ' && c2 <= '~'))
			    {
			      phase2_ungetc (c2);
			      while (--i >= 0)
				phase2_ungetc (buf[i]);
			      phase2_ungetc (c1);
			      phase2_ungetc (c);
			      ++*backslash_counter;
			      return UNICODE ('\\');
			    }
			  if (c2 == '}')
			    break;
			  buf[i] = c2;
			}
		      buf[i] = '\0';

		      n = unicode_name_character ((char *) buf);
		      if (n != UNINAME_INVALID)
			{
			  *backslash_counter = 0;
			  return UNICODE (n);
			}

		      phase2_ungetc ('}');
		      while (--i >= 0)
			phase2_ungetc (buf[i]);
		    }
		  phase2_ungetc (c1);
		  phase2_ungetc (c);
		  ++*backslash_counter;
		  return UNICODE ('\\');
		}
	    }
	}

      phase2_ungetc (c);
      ++*backslash_counter;
      return UNICODE ('\\');
    }
}


/* Combine characters into tokens.  Discard whitespace except newlines at
   the end of logical lines.  */

/* Number of pending open parentheses/braces/brackets.  */
static int open_pbb;

static token_ty phase5_pushback[1];
static int phase5_pushback_length;

static void
phase5_get (token_ty *tp)
{
  int c;

  if (phase5_pushback_length)
    {
      *tp = phase5_pushback[--phase5_pushback_length];
      return;
    }

  for (;;)
    {
      tp->line_number = line_number;
      c = phase3_getc ();

      switch (c)
	{
	case UEOF:
	  tp->type = token_type_eof;
	  return;

	case ' ':
	case '\t':
	case '\f':
	  /* Ignore whitespace and comments.  */
	  continue;

	case '\n':
	  if (last_non_comment_line > last_comment_line)
	    savable_comment_reset ();
	  /* Ignore newline if and only if it is used for implicit line
	     joining.  */
	  if (open_pbb > 0)
	    continue;
	  tp->type = token_type_other;
	  return;
	}

      last_non_comment_line = tp->line_number;

      switch (c)
	{
	case '.':
	  {
	    int c1 = phase3_getc ();
	    phase3_ungetc (c1);
	    if (!(c1 >= '0' && c1 <= '9'))
	      {

		tp->type = token_type_other;
		return;
	      }
	  }
	  /* FALLTHROUGH */
	case 'A': case 'B': case 'C': case 'D': case 'E': case 'F':
	case 'G': case 'H': case 'I': case 'J': case 'K': case 'L':
	case 'M': case 'N': case 'O': case 'P': case 'Q':
	case 'S': case 'T':           case 'V': case 'W': case 'X':
	case 'Y': case 'Z':
	case '_':
	case 'a': case 'b': case 'c': case 'd': case 'e': case 'f':
	case 'g': case 'h': case 'i': case 'j': case 'k': case 'l':
	case 'm': case 'n': case 'o': case 'p': case 'q':
	case 's': case 't':           case 'v': case 'w': case 'x':
	case 'y': case 'z':
	case '0': case '1': case '2': case '3': case '4':
	case '5': case '6': case '7': case '8': case '9':
	symbol:
	  /* Symbol, or part of a number.  */
	  {
	    static char *buffer;
	    static int bufmax;
	    int bufpos;

	    bufpos = 0;
	    for (;;)
	      {
		if (bufpos >= bufmax)
		  {
		    bufmax = 2 * bufmax + 10;
		    buffer = xrealloc (buffer, bufmax);
		  }
		buffer[bufpos++] = c;
		c = phase3_getc ();
		switch (c)
		  {
		  case 'A': case 'B': case 'C': case 'D': case 'E': case 'F':
		  case 'G': case 'H': case 'I': case 'J': case 'K': case 'L':
		  case 'M': case 'N': case 'O': case 'P': case 'Q': case 'R':
		  case 'S': case 'T': case 'U': case 'V': case 'W': case 'X':
		  case 'Y': case 'Z':
		  case '_':
		  case 'a': case 'b': case 'c': case 'd': case 'e': case 'f':
		  case 'g': case 'h': case 'i': case 'j': case 'k': case 'l':
		  case 'm': case 'n': case 'o': case 'p': case 'q': case 'r':
		  case 's': case 't': case 'u': case 'v': case 'w': case 'x':
		  case 'y': case 'z':
		  case '0': case '1': case '2': case '3': case '4':
		  case '5': case '6': case '7': case '8': case '9':
		    continue;
		  default:
		    phase3_ungetc (c);
		    break;
		  }
		break;
	      }
	    if (bufpos >= bufmax)
	      {
		bufmax = 2 * bufmax + 10;
		buffer = xrealloc (buffer, bufmax);
	      }
	    buffer[bufpos] = '\0';
	    tp->string = xstrdup (buffer);
	    tp->type = token_type_symbol;
	    return;
	  }

	/* Strings.  */
	  {
	    struct mixed_string_buffer literal;
	    int quote_char;
	    bool interpret_ansic;
	    bool interpret_unicode;
	    bool triple;
	    unsigned int backslash_counter;

	    case 'R': case 'r':
	      {
		int c1 = phase2_getc ();
		if (c1 == '"' || c1 == '\'')
		  {
		    quote_char = c1;
		    interpret_ansic = false;
		    interpret_unicode = false;
		    goto string;
		  }
		phase2_ungetc (c1);
		goto symbol;
	      }

	    case 'U': case 'u':
	      {
		int c1 = phase2_getc ();
		if (c1 == '"' || c1 == '\'')
		  {
		    quote_char = c1;
		    interpret_ansic = true;
		    interpret_unicode = true;
		    goto string;
		  }
		if (c1 == 'R' || c1 == 'r')
		  {
		    int c2 = phase2_getc ();
		    if (c2 == '"' || c2 == '\'')
		      {
			quote_char = c2;
			interpret_ansic = false;
			interpret_unicode = true;
			goto string;
		      }
		    phase2_ungetc (c2);
		  }
		phase2_ungetc (c1);
		goto symbol;
	      }

	    case '"': case '\'':
	      quote_char = c;
	      interpret_ansic = true;
	      interpret_unicode = false;
	    string:
	      triple = false;
	      {
		int c1 = phase2_getc ();
		if (c1 == quote_char)
		  {
		    int c2 = phase2_getc ();
		    if (c2 == quote_char)
		      triple = true;
		    else
		      {
			phase2_ungetc (c2);
			phase2_ungetc (c1);
		      }
		  }
		else
		  phase2_ungetc (c1);
	      }
	      backslash_counter = 0;
	      /* Start accumulating the string.  */
	      init_mixed_string_buffer (&literal);
	      for (;;)
		{
		  int uc = phase7_getuc (quote_char, triple, interpret_ansic,
					 interpret_unicode, &backslash_counter);

		  if (uc == P7_EOF || uc == P7_STRING_END)
		    break;

		  if (IS_UNICODE (uc))
		    assert (UNICODE_VALUE (uc) >= 0
			    && UNICODE_VALUE (uc) < 0x110000);

		  mixed_string_buffer_append (&literal, uc);
		}
	      tp->string = xstrdup (mixed_string_buffer_result (&literal));
	      free_mixed_string_buffer (&literal);
	      tp->comment = add_reference (savable_comment);
	      tp->type = token_type_string;
	      return;
	  }

	case '(':
	  open_pbb++;
	  tp->type = token_type_lparen;
	  return;

	case ')':
	  if (open_pbb > 0)
	    open_pbb--;
	  tp->type = token_type_rparen;
	  return;

	case ',':
	  tp->type = token_type_comma;
	  return;

	case '[': case '{':
	  open_pbb++;
	  tp->type = token_type_other;
	  return;

	case ']': case '}':
	  if (open_pbb > 0)
	    open_pbb--;
	  tp->type = token_type_other;
	  return;

	default:
	  /* We could carefully recognize each of the 2 and 3 character
	     operators, but it is not necessary, as we only need to recognize
	     gettext invocations.  Don't bother.  */
	  tp->type = token_type_other;
	  return;
	}
    }
}

/* Supports only one pushback token.  */
static void
phase5_unget (token_ty *tp)
{
  if (tp->type != token_type_eof)
    {
      if (phase5_pushback_length == SIZEOF (phase5_pushback))
	abort ();
      phase5_pushback[phase5_pushback_length++] = *tp;
    }
}


/* Combine adjacent strings to form a single string.  Note that the end
   of a logical line appears as a token of its own, therefore strings that
   belong to different logical lines will not be concatenated.  */

static void
x_python_lex (token_ty *tp)
{
  phase5_get (tp);
  if (tp->type != token_type_string)
    return;
  for (;;)
    {
      token_ty tmp;
      size_t len;

      phase5_get (&tmp);
      if (tmp.type != token_type_string)
	{
	  phase5_unget (&tmp);
	  return;
	}
      len = strlen (tp->string);
      tp->string = xrealloc (tp->string, len + strlen (tmp.string) + 1);
      strcpy (tp->string + len, tmp.string);
      free (tmp.string);
    }
}


/* ========================= Extracting strings.  ========================== */


/* Context lookup table.  */
static flag_context_list_table_ty *flag_context_list_table;


/* The file is broken into tokens.  Scan the token stream, looking for
   a keyword, followed by a left paren, followed by a string.  When we
   see this sequence, we have something to remember.  We assume we are
   looking at a valid C or C++ program, and leave the complaints about
   the grammar to the compiler.

     Normal handling: Look for
       keyword ( ... msgid ... )
     Plural handling: Look for
       keyword ( ... msgid ... msgid_plural ... )

   We use recursion because the arguments before msgid or between msgid
   and msgid_plural can contain subexpressions of the same form.  */


/* Extract messages until the next balanced closing parenthesis.
   Extracted messages are added to MLP.
   Return true upon eof, false upon closing parenthesis.  */
static bool
extract_parenthesized (message_list_ty *mlp,
		       flag_context_ty outer_context,
		       flag_context_list_iterator_ty context_iter,
		       struct arglist_parser *argparser)
{
  /* Current argument number.  */
  int arg = 1;
  /* 0 when no keyword has been seen.  1 right after a keyword is seen.  */
  int state;
  /* Parameters of the keyword just seen.  Defined only in state 1.  */
  const struct callshapes *next_shapes = NULL;
  /* Context iterator that will be used if the next token is a '('.  */
  flag_context_list_iterator_ty next_context_iter =
    passthrough_context_list_iterator;
  /* Current context.  */
  flag_context_ty inner_context =
    inherited_context (outer_context,
		       flag_context_list_iterator_advance (&context_iter));

  /* Start state is 0.  */
  state = 0;

  for (;;)
    {
      token_ty token;

      x_python_lex (&token);
      switch (token.type)
	{
	case token_type_symbol:
	  {
	    void *keyword_value;

	    if (hash_find_entry (&keywords, token.string, strlen (token.string),
				 &keyword_value)
		== 0)
	      {
		next_shapes = (const struct callshapes *) keyword_value;
		state = 1;
	      }
	    else
	      state = 0;
	  }
	  next_context_iter =
	    flag_context_list_iterator (
	      flag_context_list_table_lookup (
		flag_context_list_table,
		token.string, strlen (token.string)));
	  free (token.string);
	  continue;

	case token_type_lparen:
	  if (extract_parenthesized (mlp, inner_context, next_context_iter,
				     arglist_parser_alloc (mlp,
							   state ? next_shapes : NULL)))
	    {
	      xgettext_current_source_encoding = po_charset_utf8;
	      arglist_parser_done (argparser, arg);
	      xgettext_current_source_encoding = xgettext_current_file_source_encoding;
	      return true;
	    }
	  next_context_iter = null_context_list_iterator;
	  state = 0;
	  continue;

	case token_type_rparen:
	  xgettext_current_source_encoding = po_charset_utf8;
	  arglist_parser_done (argparser, arg);
	  xgettext_current_source_encoding = xgettext_current_file_source_encoding;
	  return false;

	case token_type_comma:
	  arg++;
	  inner_context =
	    inherited_context (outer_context,
			       flag_context_list_iterator_advance (
				 &context_iter));
	  next_context_iter = passthrough_context_list_iterator;
	  state = 0;
	  continue;

	case token_type_string:
	  {
	    lex_pos_ty pos;
	    pos.file_name = logical_file_name;
	    pos.line_number = token.line_number;

	    xgettext_current_source_encoding = po_charset_utf8;
	    if (extract_all)
	      remember_a_message (mlp, NULL, token.string, inner_context,
				  &pos, token.comment);
	    else
	      arglist_parser_remember (argparser, arg, token.string,
				       inner_context,
				       pos.file_name, pos.line_number,
				       token.comment);
	    xgettext_current_source_encoding = xgettext_current_file_source_encoding;
	  }
	  drop_reference (token.comment);
	  next_context_iter = null_context_list_iterator;
	  state = 0;
	  continue;

	case token_type_eof:
	  xgettext_current_source_encoding = po_charset_utf8;
	  arglist_parser_done (argparser, arg);
	  xgettext_current_source_encoding = xgettext_current_file_source_encoding;
	  return true;

	case token_type_other:
	  next_context_iter = null_context_list_iterator;
	  state = 0;
	  continue;

	default:
	  abort ();
	}
    }
}


void
extract_python (FILE *f,
		const char *real_filename, const char *logical_filename,
		flag_context_list_table_ty *flag_table,
		msgdomain_list_ty *mdlp)
{
  message_list_ty *mlp = mdlp->item[0]->messages;

  fp = f;
  real_file_name = real_filename;
  logical_file_name = xstrdup (logical_filename);
  line_number = 1;

  last_comment_line = -1;
  last_non_comment_line = -1;

  xgettext_current_file_source_encoding = xgettext_global_source_encoding;
#if HAVE_ICONV
  xgettext_current_file_source_iconv = xgettext_global_source_iconv;
#endif

  xgettext_current_source_encoding = xgettext_current_file_source_encoding;
#if HAVE_ICONV
  xgettext_current_source_iconv = xgettext_current_file_source_iconv;
#endif

  continuation_or_nonblank_line = false;

  open_pbb = 0;

  flag_context_list_table = flag_table;

  init_keywords ();

  /* Eat tokens until eof is seen.  When extract_parenthesized returns
     due to an unbalanced closing parenthesis, just restart it.  */
  while (!extract_parenthesized (mlp, null_context, null_context_list_iterator,
				 arglist_parser_alloc (mlp, NULL)))
    ;

  fp = NULL;
  real_file_name = NULL;
  logical_file_name = NULL;
  line_number = 0;
}