1 /* csv module */

     2 

     3 /*

     4 

     5 This module provides the low-level underpinnings of a CSV reading/writing

     6 module.  Users should not use this module directly, but import the csv.py

     7 module instead.

     8 

     9 **** For people modifying this code, please note that as of this writing

    10 **** (2003-03-23), it is intended that this code should work with Python

    11 **** 2.2.

    12 

    13 */

    14 

    15 #define MODULE_VERSION "1.0"

    16 

    17 #include "Python.h"

    18 #include "structmember.h"

    19 

    20 

    21 /* begin 2.2 compatibility macros */

    22 #ifndef PyDoc_STRVAR

    23 /* Define macros for inline documentation. */

    24 #define PyDoc_VAR(name) static char name[]

    25 #define PyDoc_STRVAR(name,str) PyDoc_VAR(name) = PyDoc_STR(str)

    26 #ifdef WITH_DOC_STRINGS

    27 #define PyDoc_STR(str) str

    28 #else

    29 #define PyDoc_STR(str) ""

    30 #endif

    31 #endif /* ifndef PyDoc_STRVAR */

    32 

    33 #ifndef PyMODINIT_FUNC

    34 #	if defined(__cplusplus)

    35 #		define PyMODINIT_FUNC extern "C" void

    36 #	else /* __cplusplus */

    37 #		define PyMODINIT_FUNC void

    38 #	endif /* __cplusplus */

    39 #endif

    40 

    41 #ifndef Py_CLEAR

    42 #define Py_CLEAR(op)						\

    43 	do {							\

    44 		if (op) {					\

    45 			PyObject *tmp = (PyObject *)(op);	\

    46 			(op) = NULL;				\

    47 			Py_DECREF(tmp);				\

    48 		}						\

    49 	} while (0)

    50 #endif

    51 #ifndef Py_VISIT

    52 #define Py_VISIT(op)							\

    53         do { 								\

    54                 if (op) {						\

    55                         int vret = visit((PyObject *)(op), arg);	\

    56                         if (vret)					\

    57                                 return vret;				\

    58                 }							\

    59         } while (0)

    60 #endif

    61 

    62 /* end 2.2 compatibility macros */

    63 

    64 #define IS_BASESTRING(o) \

    65 	PyObject_TypeCheck(o, &PyBaseString_Type)

    66 

    67 static PyObject *error_obj;	/* CSV exception */

    68 static PyObject *dialects;      /* Dialect registry */

    69 static long field_limit = 128 * 1024;	/* max parsed field size */

    70 

    71 typedef enum {

    72 	START_RECORD, START_FIELD, ESCAPED_CHAR, IN_FIELD, 

    73 	IN_QUOTED_FIELD, ESCAPE_IN_QUOTED_FIELD, QUOTE_IN_QUOTED_FIELD,

    74 	EAT_CRNL

    75 } ParserState;

    76 

    77 typedef enum {

    78 	QUOTE_MINIMAL, QUOTE_ALL, QUOTE_NONNUMERIC, QUOTE_NONE

    79 } QuoteStyle;

    80 

    81 typedef struct {

    82 	QuoteStyle style;

    83 	char *name;

    84 } StyleDesc;

    85 

    86 static StyleDesc quote_styles[] = {

    87 	{ QUOTE_MINIMAL,    "QUOTE_MINIMAL" },

    88 	{ QUOTE_ALL,        "QUOTE_ALL" },

    89 	{ QUOTE_NONNUMERIC, "QUOTE_NONNUMERIC" },

    90 	{ QUOTE_NONE,       "QUOTE_NONE" },

    91 	{ 0 }

    92 };

    93 

    94 typedef struct {

    95         PyObject_HEAD

    96         

    97 	int doublequote;	/* is " represented by ""? */

    98 	char delimiter;		/* field separator */

    99 	char quotechar;		/* quote character */

   100 	char escapechar;	/* escape character */

   101 	int skipinitialspace;	/* ignore spaces following delimiter? */

   102 	PyObject *lineterminator; /* string to write between records */

   103 	int quoting;		/* style of quoting to write */

   104 

   105 	int strict;		/* raise exception on bad CSV */

   106 } DialectObj;

   107 

   108 staticforward PyTypeObject Dialect_Type;

   109 

   110 typedef struct {

   111         PyObject_HEAD

   112 

   113         PyObject *input_iter;   /* iterate over this for input lines */

   114 

   115         DialectObj *dialect;    /* parsing dialect */

   116 

   117 	PyObject *fields;	/* field list for current record */

   118 	ParserState state;	/* current CSV parse state */

   119 	char *field;		/* build current field in here */

   120 	int field_size;		/* size of allocated buffer */

   121 	int field_len;		/* length of current field */

   122 	int numeric_field;	/* treat field as numeric */

   123 	unsigned long line_num;	/* Source-file line number */

   124 } ReaderObj;

   125 

   126 staticforward PyTypeObject Reader_Type;

   127 

   128 #define ReaderObject_Check(v)   (Py_TYPE(v) == &Reader_Type)

   129 

   130 typedef struct {

   131         PyObject_HEAD

   132 

   133         PyObject *writeline;    /* write output lines to this file */

   134 

   135         DialectObj *dialect;    /* parsing dialect */

   136 

   137 	char *rec;		/* buffer for parser.join */

   138 	int rec_size;		/* size of allocated record */

   139 	int rec_len;		/* length of record */

   140 	int num_fields;		/* number of fields in record */

   141 } WriterObj;        

   142 

   143 staticforward PyTypeObject Writer_Type;

   144 

   145 /*

   146  * DIALECT class

   147  */

   148 

   149 static PyObject *

   150 get_dialect_from_registry(PyObject * name_obj)

   151 {

   152         PyObject *dialect_obj;

   153 

   154         dialect_obj = PyDict_GetItem(dialects, name_obj);

   155 	if (dialect_obj == NULL) {

   156 		if (!PyErr_Occurred())

   157 			PyErr_Format(error_obj, "unknown dialect");

   158 	}

   159 	else

   160 		Py_INCREF(dialect_obj);

   161         return dialect_obj;

   162 }

   163 

   164 static PyObject *

   165 get_string(PyObject *str)

   166 {

   167         Py_XINCREF(str);

   168         return str;

   169 }

   170 

   171 static PyObject *

   172 get_nullchar_as_None(char c)

   173 {

   174         if (c == '\0') {

   175                 Py_INCREF(Py_None);

   176                 return Py_None;

   177         }

   178         else

   179                 return PyString_FromStringAndSize((char*)&c, 1);

   180 }

   181 

   182 static PyObject *

   183 Dialect_get_lineterminator(DialectObj *self)

   184 {

   185         return get_string(self->lineterminator);

   186 }

   187 

   188 static PyObject *

   189 Dialect_get_escapechar(DialectObj *self)

   190 {

   191         return get_nullchar_as_None(self->escapechar);

   192 }

   193 

   194 static PyObject *

   195 Dialect_get_quotechar(DialectObj *self)

   196 {

   197         return get_nullchar_as_None(self->quotechar);

   198 }

   199 

   200 static PyObject *

   201 Dialect_get_quoting(DialectObj *self)

   202 {

   203         return PyInt_FromLong(self->quoting);

   204 }

   205 

   206 static int

   207 _set_bool(const char *name, int *target, PyObject *src, int dflt)

   208 {

   209 	if (src == NULL)

   210 		*target = dflt;

   211 	else

   212 		*target = PyObject_IsTrue(src);

   213 	return 0;

   214 }

   215 

   216 static int

   217 _set_int(const char *name, int *target, PyObject *src, int dflt)

   218 {

   219 	if (src == NULL)

   220 		*target = dflt;

   221 	else {

   222 		if (!PyInt_Check(src)) {

   223 			PyErr_Format(PyExc_TypeError, 

   224 				     "\"%s\" must be an integer", name);

   225 			return -1;

   226 		}

   227 		*target = PyInt_AsLong(src);

   228 	}

   229 	return 0;

   230 }

   231 

   232 static int

   233 _set_char(const char *name, char *target, PyObject *src, char dflt)

   234 {

   235 	if (src == NULL)

   236 		*target = dflt;

   237 	else {

   238 		if (src == Py_None || PyString_Size(src) == 0)

   239 			*target = '\0';

   240 		else if (!PyString_Check(src) || PyString_Size(src) != 1) {

   241 			PyErr_Format(PyExc_TypeError, 

   242 				     "\"%s\" must be an 1-character string", 

   243 				     name);

   244 			return -1;

   245 		}

   246 		else {

   247 			char *s = PyString_AsString(src);

   248 			if (s == NULL)

   249 				return -1;

   250 			*target = s[0];

   251 		}

   252 	}

   253         return 0;

   254 }

   255 

   256 static int

   257 _set_str(const char *name, PyObject **target, PyObject *src, const char *dflt)

   258 {

   259 	if (src == NULL)

   260 		*target = PyString_FromString(dflt);

   261 	else {

   262 		if (src == Py_None)

   263 			*target = NULL;

   264 		else if (!IS_BASESTRING(src)) {

   265 			PyErr_Format(PyExc_TypeError, 

   266 				     "\"%s\" must be an string", name);

   267 			return -1;

   268 		}

   269 		else {

   270 			Py_XDECREF(*target);

   271 			Py_INCREF(src);

   272 			*target = src;

   273 		}

   274 	}

   275         return 0;

   276 }

   277 

   278 static int

   279 dialect_check_quoting(int quoting)

   280 {

   281         StyleDesc *qs = quote_styles;

   282 

   283 	for (qs = quote_styles; qs->name; qs++) {

   284 		if (qs->style == quoting)

   285                         return 0;

   286         }

   287 	PyErr_Format(PyExc_TypeError, "bad \"quoting\" value");

   288         return -1;

   289 }

   290 

   291 #define D_OFF(x) offsetof(DialectObj, x)

   292 

   293 static struct PyMemberDef Dialect_memberlist[] = {

   294 	{ "delimiter",          T_CHAR, D_OFF(delimiter), READONLY },

   295 	{ "skipinitialspace",   T_INT, D_OFF(skipinitialspace), READONLY },

   296 	{ "doublequote",        T_INT, D_OFF(doublequote), READONLY },

   297 	{ "strict",             T_INT, D_OFF(strict), READONLY },

   298 	{ NULL }

   299 };

   300 

   301 static PyGetSetDef Dialect_getsetlist[] = {

   302 	{ "escapechar",		(getter)Dialect_get_escapechar},

   303 	{ "lineterminator",	(getter)Dialect_get_lineterminator},

   304 	{ "quotechar",		(getter)Dialect_get_quotechar},

   305 	{ "quoting",		(getter)Dialect_get_quoting},

   306 	{NULL},

   307 };

   308 

   309 static void

   310 Dialect_dealloc(DialectObj *self)

   311 {

   312         Py_XDECREF(self->lineterminator);

   313         Py_TYPE(self)->tp_free((PyObject *)self);

   314 }

   315 

   316 static char *dialect_kws[] = {

   317 	"dialect",

   318 	"delimiter",

   319 	"doublequote",

   320 	"escapechar",

   321 	"lineterminator",

   322 	"quotechar",

   323 	"quoting",

   324 	"skipinitialspace",

   325 	"strict",

   326 	NULL

   327 };

   328 

   329 static PyObject *

   330 dialect_new(PyTypeObject *type, PyObject *args, PyObject *kwargs)

   331 {

   332 	DialectObj *self;

   333 	PyObject *ret = NULL;

   334 	PyObject *dialect = NULL;

   335 	PyObject *delimiter = NULL;

   336 	PyObject *doublequote = NULL;

   337 	PyObject *escapechar = NULL;

   338 	PyObject *lineterminator = NULL;

   339 	PyObject *quotechar = NULL;

   340 	PyObject *quoting = NULL;

   341 	PyObject *skipinitialspace = NULL;

   342 	PyObject *strict = NULL;

   343 

   344 	if (!PyArg_ParseTupleAndKeywords(args, kwargs,

   345 					 "|OOOOOOOOO", dialect_kws,

   346 					 &dialect,

   347 					 &delimiter,

   348 					 &doublequote,

   349 					 &escapechar,

   350 					 &lineterminator,

   351 					 &quotechar,

   352 					 &quoting,

   353 					 &skipinitialspace,

   354 					 &strict))

   355 		return NULL;

   356 

   357 	if (dialect != NULL) {

   358 		if (IS_BASESTRING(dialect)) {

   359 			dialect = get_dialect_from_registry(dialect);

   360 			if (dialect == NULL)

   361 				return NULL;

   362 		}

   363 		else

   364 			Py_INCREF(dialect);

   365 		/* Can we reuse this instance? */

   366 		if (PyObject_TypeCheck(dialect, &Dialect_Type) &&

   367 		    delimiter == 0 &&

   368 		    doublequote == 0 &&

   369 		    escapechar == 0 &&

   370 		    lineterminator == 0 &&

   371 		    quotechar == 0 &&

   372 		    quoting == 0 &&

   373 		    skipinitialspace == 0 &&

   374 		    strict == 0)

   375 			return dialect;

   376 	}

   377 

   378 	self = (DialectObj *)type->tp_alloc(type, 0);

   379 	if (self == NULL) {

   380 		Py_XDECREF(dialect);

   381 		return NULL;

   382 	}

   383 	self->lineterminator = NULL;

   384 

   385 	Py_XINCREF(delimiter);

   386 	Py_XINCREF(doublequote);

   387 	Py_XINCREF(escapechar);

   388 	Py_XINCREF(lineterminator);

   389 	Py_XINCREF(quotechar);

   390 	Py_XINCREF(quoting);

   391 	Py_XINCREF(skipinitialspace);

   392 	Py_XINCREF(strict);

   393 	if (dialect != NULL) {

   394 #define DIALECT_GETATTR(v, n) \

   395 		if (v == NULL) \

   396 			v = PyObject_GetAttrString(dialect, n)

   397 		DIALECT_GETATTR(delimiter, "delimiter");

   398 		DIALECT_GETATTR(doublequote, "doublequote");

   399 		DIALECT_GETATTR(escapechar, "escapechar");

   400 		DIALECT_GETATTR(lineterminator, "lineterminator");

   401 		DIALECT_GETATTR(quotechar, "quotechar");

   402 		DIALECT_GETATTR(quoting, "quoting");

   403 		DIALECT_GETATTR(skipinitialspace, "skipinitialspace");

   404 		DIALECT_GETATTR(strict, "strict");

   405 		PyErr_Clear();

   406 	}

   407 

   408 	/* check types and convert to C values */

   409 #define DIASET(meth, name, target, src, dflt) \

   410 	if (meth(name, target, src, dflt)) \

   411 		goto err

   412 	DIASET(_set_char, "delimiter", &self->delimiter, delimiter, ',');

   413 	DIASET(_set_bool, "doublequote", &self->doublequote, doublequote, 1);

   414 	DIASET(_set_char, "escapechar", &self->escapechar, escapechar, 0);

   415 	DIASET(_set_str, "lineterminator", &self->lineterminator, lineterminator, "\r\n");

   416 	DIASET(_set_char, "quotechar", &self->quotechar, quotechar, '"');

   417 	DIASET(_set_int, "quoting", &self->quoting, quoting, QUOTE_MINIMAL);

   418 	DIASET(_set_bool, "skipinitialspace", &self->skipinitialspace, skipinitialspace, 0);

   419 	DIASET(_set_bool, "strict", &self->strict, strict, 0);

   420 

   421 	/* validate options */

   422 	if (dialect_check_quoting(self->quoting))

   423 		goto err;

   424 	if (self->delimiter == 0) {

   425                 PyErr_SetString(PyExc_TypeError, "delimiter must be set");

   426 		goto err;

   427 	}

   428 	if (quotechar == Py_None && quoting == NULL)

   429 		self->quoting = QUOTE_NONE;

   430 	if (self->quoting != QUOTE_NONE && self->quotechar == 0) {

   431                 PyErr_SetString(PyExc_TypeError, 

   432 				"quotechar must be set if quoting enabled");

   433 		goto err;

   434 	}

   435 	if (self->lineterminator == 0) {

   436 		PyErr_SetString(PyExc_TypeError, "lineterminator must be set");

   437 		goto err;

   438 	}

   439 

   440 	ret = (PyObject *)self;

   441 	Py_INCREF(self);

   442 err:

   443 	Py_XDECREF(self);

   444 	Py_XDECREF(dialect);

   445 	Py_XDECREF(delimiter);

   446 	Py_XDECREF(doublequote);

   447 	Py_XDECREF(escapechar);

   448 	Py_XDECREF(lineterminator);

   449 	Py_XDECREF(quotechar);

   450 	Py_XDECREF(quoting);

   451 	Py_XDECREF(skipinitialspace);

   452 	Py_XDECREF(strict);

   453 	return ret;

   454 }

   455 

   456 

   457 PyDoc_STRVAR(Dialect_Type_doc, 

   458 "CSV dialect\n"

   459 "\n"

   460 "The Dialect type records CSV parsing and generation options.\n");

   461 

   462 static PyTypeObject Dialect_Type = {

   463 	PyVarObject_HEAD_INIT(NULL, 0)

   464 	"_csv.Dialect",                         /* tp_name */

   465 	sizeof(DialectObj),                     /* tp_basicsize */

   466 	0,                                      /* tp_itemsize */

   467 	/*  methods  */

   468 	(destructor)Dialect_dealloc,            /* tp_dealloc */

   469 	(printfunc)0,                           /* tp_print */

   470 	(getattrfunc)0,                         /* tp_getattr */

   471 	(setattrfunc)0,                         /* tp_setattr */

   472 	(cmpfunc)0,                             /* tp_compare */

   473 	(reprfunc)0,                            /* tp_repr */

   474 	0,                                      /* tp_as_number */

   475 	0,                                      /* tp_as_sequence */

   476 	0,                                      /* tp_as_mapping */

   477 	(hashfunc)0,                            /* tp_hash */

   478 	(ternaryfunc)0,                         /* tp_call */

   479 	(reprfunc)0,                    	/* tp_str */

   480 	0,                                      /* tp_getattro */

   481         0,                                      /* tp_setattro */

   482         0,                                      /* tp_as_buffer */

   483         Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */

   484 	Dialect_Type_doc,                       /* tp_doc */

   485         0,                                      /* tp_traverse */

   486         0,                                      /* tp_clear */

   487         0,                                      /* tp_richcompare */

   488         0,                                      /* tp_weaklistoffset */

   489         0,                                      /* tp_iter */

   490         0,                                      /* tp_iternext */

   491 	0,					/* tp_methods */

   492         Dialect_memberlist,                     /* tp_members */

   493         Dialect_getsetlist,                     /* tp_getset */

   494 	0,					/* tp_base */

   495 	0,					/* tp_dict */

   496 	0,					/* tp_descr_get */

   497 	0,					/* tp_descr_set */

   498 	0,					/* tp_dictoffset */

   499 	0,					/* tp_init */

   500 	0,					/* tp_alloc */

   501 	dialect_new,			        /* tp_new */

   502 	0,                           		/* tp_free */

   503 };

   504 

   505 /*

   506  * Return an instance of the dialect type, given a Python instance or kwarg

   507  * description of the dialect

   508  */

   509 static PyObject *

   510 _call_dialect(PyObject *dialect_inst, PyObject *kwargs)

   511 {

   512 	PyObject *ctor_args;

   513 	PyObject *dialect;

   514 

   515 	ctor_args = Py_BuildValue(dialect_inst ? "(O)" : "()", dialect_inst);

   516 	if (ctor_args == NULL)

   517 		return NULL;

   518 	dialect = PyObject_Call((PyObject *)&Dialect_Type, ctor_args, kwargs);

   519 	Py_DECREF(ctor_args);

   520 	return dialect;

   521 }

   522 

   523 /*

   524  * READER

   525  */

   526 static int

   527 parse_save_field(ReaderObj *self)

   528 {

   529 	PyObject *field;

   530 

   531 	field = PyString_FromStringAndSize(self->field, self->field_len);

   532 	if (field == NULL)

   533 		return -1;

   534 	self->field_len = 0;

   535 	if (self->numeric_field) {

   536 		PyObject *tmp;

   537 

   538 		self->numeric_field = 0;

   539 		tmp = PyNumber_Float(field);

   540 		if (tmp == NULL) {

   541 			Py_DECREF(field);

   542 			return -1;

   543 		}

   544 		Py_DECREF(field);

   545 		field = tmp;

   546 	}

   547 	PyList_Append(self->fields, field);

   548 	Py_DECREF(field);

   549 	return 0;

   550 }

   551 

   552 static int

   553 parse_grow_buff(ReaderObj *self)

   554 {

   555 	if (self->field_size == 0) {

   556 		self->field_size = 4096;

   557 		if (self->field != NULL)

   558 			PyMem_Free(self->field);

   559 		self->field = PyMem_Malloc(self->field_size);

   560 	}

   561 	else {

   562 		if (self->field_size > INT_MAX / 2) {

   563 			PyErr_NoMemory();

   564 			return 0;

   565 		} 

   566 		self->field_size *= 2;

   567 		self->field = PyMem_Realloc(self->field, self->field_size);

   568 	}

   569 	if (self->field == NULL) {

   570 		PyErr_NoMemory();

   571 		return 0;

   572 	}

   573 	return 1;

   574 }

   575 

   576 static int

   577 parse_add_char(ReaderObj *self, char c)

   578 {

   579 	if (self->field_len >= field_limit) {

   580 		PyErr_Format(error_obj, "field larger than field limit (%ld)",

   581 			     field_limit);

   582 		return -1;

   583 	}

   584 	if (self->field_len == self->field_size && !parse_grow_buff(self))

   585 		return -1;

   586 	self->field[self->field_len++] = c;

   587 	return 0;

   588 }

   589 

   590 static int

   591 parse_process_char(ReaderObj *self, char c)

   592 {

   593         DialectObj *dialect = self->dialect;

   594 

   595 	switch (self->state) {

   596 	case START_RECORD:

   597 		/* start of record */

   598 		if (c == '\0')

   599 			/* empty line - return [] */

   600 			break;

   601 		else if (c == '\n' || c == '\r') {

   602 			self->state = EAT_CRNL;

   603 			break;

   604 		}

   605 		/* normal character - handle as START_FIELD */

   606 		self->state = START_FIELD;

   607 		/* fallthru */

   608 	case START_FIELD:

   609 		/* expecting field */

   610 		if (c == '\n' || c == '\r' || c == '\0') {

   611 			/* save empty field - return [fields] */

   612 			if (parse_save_field(self) < 0)

   613 				return -1;

   614 			self->state = (c == '\0' ? START_RECORD : EAT_CRNL);

   615 		}

   616 		else if (c == dialect->quotechar && 

   617 			 dialect->quoting != QUOTE_NONE) {

   618 			/* start quoted field */

   619 			self->state = IN_QUOTED_FIELD;

   620 		}

   621 		else if (c == dialect->escapechar) {

   622 			/* possible escaped character */

   623 			self->state = ESCAPED_CHAR;

   624 		}

   625 		else if (c == ' ' && dialect->skipinitialspace)

   626 			/* ignore space at start of field */

   627 			;

   628 		else if (c == dialect->delimiter) {

   629 			/* save empty field */

   630 			if (parse_save_field(self) < 0)

   631 				return -1;

   632 		}

   633 		else {

   634 			/* begin new unquoted field */

   635 			if (dialect->quoting == QUOTE_NONNUMERIC)

   636 				self->numeric_field = 1;

   637 			if (parse_add_char(self, c) < 0)

   638 				return -1;

   639 			self->state = IN_FIELD;

   640 		}

   641 		break;

   642 

   643 	case ESCAPED_CHAR:

   644 		if (c == '\0')

   645 			c = '\n';

   646 		if (parse_add_char(self, c) < 0)

   647 			return -1;

   648 		self->state = IN_FIELD;

   649 		break;

   650 

   651 	case IN_FIELD:

   652 		/* in unquoted field */

   653 		if (c == '\n' || c == '\r' || c == '\0') {

   654 			/* end of line - return [fields] */

   655 			if (parse_save_field(self) < 0)

   656 				return -1;

   657 			self->state = (c == '\0' ? START_RECORD : EAT_CRNL);

   658 		}

   659 		else if (c == dialect->escapechar) {

   660 			/* possible escaped character */

   661 			self->state = ESCAPED_CHAR;

   662 		}

   663 		else if (c == dialect->delimiter) {

   664 			/* save field - wait for new field */

   665 			if (parse_save_field(self) < 0)

   666 				return -1;

   667 			self->state = START_FIELD;

   668 		}

   669 		else {

   670 			/* normal character - save in field */

   671 			if (parse_add_char(self, c) < 0)

   672 				return -1;

   673 		}

   674 		break;

   675 

   676 	case IN_QUOTED_FIELD:

   677 		/* in quoted field */

   678 		if (c == '\0')

   679 			;

   680 		else if (c == dialect->escapechar) {

   681 			/* Possible escape character */

   682 			self->state = ESCAPE_IN_QUOTED_FIELD;

   683 		}

   684 		else if (c == dialect->quotechar &&

   685 			 dialect->quoting != QUOTE_NONE) {

   686 			if (dialect->doublequote) {

   687 				/* doublequote; " represented by "" */

   688 				self->state = QUOTE_IN_QUOTED_FIELD;

   689 			}

   690 			else {

   691 				/* end of quote part of field */

   692 				self->state = IN_FIELD;

   693 			}

   694 		}

   695 		else {

   696 			/* normal character - save in field */

   697 			if (parse_add_char(self, c) < 0)

   698 				return -1;

   699 		}

   700 		break;

   701 

   702 	case ESCAPE_IN_QUOTED_FIELD:

   703 		if (c == '\0')

   704 			c = '\n';

   705 		if (parse_add_char(self, c) < 0)

   706 			return -1;

   707 		self->state = IN_QUOTED_FIELD;

   708 		break;

   709 

   710 	case QUOTE_IN_QUOTED_FIELD:

   711 		/* doublequote - seen a quote in an quoted field */

   712 		if (dialect->quoting != QUOTE_NONE && 

   713                     c == dialect->quotechar) {

   714 			/* save "" as " */

   715 			if (parse_add_char(self, c) < 0)

   716 				return -1;

   717 			self->state = IN_QUOTED_FIELD;

   718 		}

   719 		else if (c == dialect->delimiter) {

   720 			/* save field - wait for new field */

   721 			if (parse_save_field(self) < 0)

   722 				return -1;

   723 			self->state = START_FIELD;

   724 		}

   725 		else if (c == '\n' || c == '\r' || c == '\0') {

   726 			/* end of line - return [fields] */

   727 			if (parse_save_field(self) < 0)

   728 				return -1;

   729 			self->state = (c == '\0' ? START_RECORD : EAT_CRNL);

   730 		}

   731 		else if (!dialect->strict) {

   732 			if (parse_add_char(self, c) < 0)

   733 				return -1;

   734 			self->state = IN_FIELD;

   735 		}

   736 		else {

   737 			/* illegal */

   738 			PyErr_Format(error_obj, "'%c' expected after '%c'", 

   739 					dialect->delimiter, 

   740                                         dialect->quotechar);

   741 			return -1;

   742 		}

   743 		break;

   744 

   745 	case EAT_CRNL:

   746 		if (c == '\n' || c == '\r')

   747 			;

   748 		else if (c == '\0')

   749 			self->state = START_RECORD;

   750 		else {

   751 			PyErr_Format(error_obj, "new-line character seen in unquoted field - do you need to open the file in universal-newline mode?");

   752 			return -1;

   753 		}

   754 		break;

   755 

   756 	}

   757 	return 0;

   758 }

   759 

   760 static int

   761 parse_reset(ReaderObj *self)

   762 {

   763 	Py_XDECREF(self->fields);

   764 	self->fields = PyList_New(0);

   765 	if (self->fields == NULL)

   766 		return -1;

   767 	self->field_len = 0;

   768 	self->state = START_RECORD;

   769 	self->numeric_field = 0;

   770 	return 0;

   771 }

   772 

   773 static PyObject *

   774 Reader_iternext(ReaderObj *self)

   775 {

   776         PyObject *lineobj;

   777         PyObject *fields = NULL;

   778         char *line, c;

   779 	int linelen;

   780 

   781 	if (parse_reset(self) < 0)

   782 		return NULL;

   783         do {

   784                 lineobj = PyIter_Next(self->input_iter);

   785                 if (lineobj == NULL) {

   786                         /* End of input OR exception */

   787                         if (!PyErr_Occurred() && self->field_len != 0)

   788                                 PyErr_Format(error_obj,

   789 					     "newline inside string");

   790                         return NULL;

   791                 }

   792 		++self->line_num;

   793 

   794                 line = PyString_AsString(lineobj);

   795 		linelen = PyString_Size(lineobj);

   796 

   797                 if (line == NULL || linelen < 0) {

   798                         Py_DECREF(lineobj);

   799                         return NULL;

   800                 }

   801                 while (linelen--) {

   802 			c = *line++;

   803 			if (c == '\0') {

   804 				Py_DECREF(lineobj);

   805 				PyErr_Format(error_obj,

   806 					     "line contains NULL byte");

   807 				goto err;

   808 			}

   809 			if (parse_process_char(self, c) < 0) {

   810 				Py_DECREF(lineobj);

   811 				goto err;

   812 			}

   813 		}

   814                 Py_DECREF(lineobj);

   815 		if (parse_process_char(self, 0) < 0)

   816 			goto err;

   817         } while (self->state != START_RECORD);

   818 

   819         fields = self->fields;

   820         self->fields = NULL;

   821 err:

   822         return fields;

   823 }

   824 

   825 static void

   826 Reader_dealloc(ReaderObj *self)

   827 {

   828 	PyObject_GC_UnTrack(self);

   829         Py_XDECREF(self->dialect);

   830         Py_XDECREF(self->input_iter);

   831         Py_XDECREF(self->fields);

   832         if (self->field != NULL)

   833         	PyMem_Free(self->field);

   834 	PyObject_GC_Del(self);

   835 }

   836 

   837 static int

   838 Reader_traverse(ReaderObj *self, visitproc visit, void *arg)

   839 {

   840 	Py_VISIT(self->dialect);

   841 	Py_VISIT(self->input_iter);

   842 	Py_VISIT(self->fields);

   843 	return 0;

   844 }

   845 

   846 static int

   847 Reader_clear(ReaderObj *self)

   848 {

   849 	Py_CLEAR(self->dialect);

   850 	Py_CLEAR(self->input_iter);

   851 	Py_CLEAR(self->fields);

   852 	return 0;

   853 }

   854 

   855 PyDoc_STRVAR(Reader_Type_doc,

   856 "CSV reader\n"

   857 "\n"

   858 "Reader objects are responsible for reading and parsing tabular data\n"

   859 "in CSV format.\n"

   860 );

   861 

   862 static struct PyMethodDef Reader_methods[] = {

   863 	{ NULL, NULL }

   864 };

   865 #define R_OFF(x) offsetof(ReaderObj, x)

   866 

   867 static struct PyMemberDef Reader_memberlist[] = {

   868 	{ "dialect", T_OBJECT, R_OFF(dialect), RO },

   869 	{ "line_num", T_ULONG, R_OFF(line_num), RO },

   870 	{ NULL }

   871 };

   872 

   873 

   874 static PyTypeObject Reader_Type = {

   875 	PyVarObject_HEAD_INIT(NULL, 0)

   876 	"_csv.reader",                          /*tp_name*/

   877 	sizeof(ReaderObj),                      /*tp_basicsize*/

   878 	0,                                      /*tp_itemsize*/

   879 	/* methods */

   880 	(destructor)Reader_dealloc,             /*tp_dealloc*/

   881 	(printfunc)0,                           /*tp_print*/

   882 	(getattrfunc)0,                         /*tp_getattr*/

   883 	(setattrfunc)0,                         /*tp_setattr*/

   884 	(cmpfunc)0,                             /*tp_compare*/

   885 	(reprfunc)0,                            /*tp_repr*/

   886 	0,                                      /*tp_as_number*/

   887 	0,                                      /*tp_as_sequence*/

   888 	0,                                      /*tp_as_mapping*/

   889 	(hashfunc)0,                            /*tp_hash*/

   890 	(ternaryfunc)0,                         /*tp_call*/

   891 	(reprfunc)0,                    	/*tp_str*/

   892 	0,                                      /*tp_getattro*/

   893         0,                                      /*tp_setattro*/

   894         0,                                      /*tp_as_buffer*/

   895         Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE |

   896 		Py_TPFLAGS_HAVE_GC,		/*tp_flags*/

   897 	Reader_Type_doc,                        /*tp_doc*/

   898         (traverseproc)Reader_traverse,          /*tp_traverse*/

   899         (inquiry)Reader_clear,                  /*tp_clear*/

   900         0,                                      /*tp_richcompare*/

   901         0,                                      /*tp_weaklistoffset*/

   902         PyObject_SelfIter,		        /*tp_iter*/

   903         (getiterfunc)Reader_iternext,           /*tp_iternext*/

   904         Reader_methods,                         /*tp_methods*/

   905         Reader_memberlist,                      /*tp_members*/

   906         0,                                      /*tp_getset*/

   907 

   908 };

   909 

   910 static PyObject *

   911 csv_reader(PyObject *module, PyObject *args, PyObject *keyword_args)

   912 {

   913 	PyObject * iterator, * dialect = NULL;

   914         ReaderObj * self = PyObject_GC_New(ReaderObj, &Reader_Type);

   915 

   916         if (!self)

   917                 return NULL;

   918 

   919         self->dialect = NULL;

   920         self->fields = NULL;

   921         self->input_iter = NULL;

   922 	self->field = NULL;

   923 	self->field_size = 0;

   924 	self->line_num = 0;

   925 

   926 	if (parse_reset(self) < 0) {

   927                 Py_DECREF(self);

   928                 return NULL;

   929 	}

   930 

   931 	if (!PyArg_UnpackTuple(args, "", 1, 2, &iterator, &dialect)) {

   932                 Py_DECREF(self);

   933                 return NULL;

   934         }

   935         self->input_iter = PyObject_GetIter(iterator);

   936         if (self->input_iter == NULL) {

   937                 PyErr_SetString(PyExc_TypeError, 

   938                                 "argument 1 must be an iterator");

   939                 Py_DECREF(self);

   940                 return NULL;

   941         }

   942 	self->dialect = (DialectObj *)_call_dialect(dialect, keyword_args);

   943         if (self->dialect == NULL) {

   944                 Py_DECREF(self);

   945                 return NULL;

   946         }

   947 

   948 	PyObject_GC_Track(self);

   949         return (PyObject *)self;

   950 }

   951 

   952 /*

   953  * WRITER

   954  */

   955 /* ---------------------------------------------------------------- */

   956 static void

   957 join_reset(WriterObj *self)

   958 {

   959 	self->rec_len = 0;

   960 	self->num_fields = 0;

   961 }

   962 

   963 #define MEM_INCR 32768

   964 

   965 /* Calculate new record length or append field to record.  Return new

   966  * record length.

   967  */

   968 static int

   969 join_append_data(WriterObj *self, char *field, int quote_empty,

   970 		 int *quoted, int copy_phase)

   971 {

   972         DialectObj *dialect = self->dialect;

   973 	int i, rec_len;

   974 	char *lineterm;

   975 

   976 #define ADDCH(c) \

   977 	do {\

   978 		if (copy_phase) \

   979 			self->rec[rec_len] = c;\

   980 		rec_len++;\

   981 	} while(0)

   982 

   983 	lineterm = PyString_AsString(dialect->lineterminator);

   984 	if (lineterm == NULL)

   985 		return -1;

   986 

   987 	rec_len = self->rec_len;

   988 

   989 	/* If this is not the first field we need a field separator */

   990 	if (self->num_fields > 0)

   991 		ADDCH(dialect->delimiter);

   992 

   993 	/* Handle preceding quote */

   994 	if (copy_phase && *quoted)

   995 		ADDCH(dialect->quotechar);

   996 

   997 	/* Copy/count field data */

   998 	for (i = 0;; i++) {

   999 		char c = field[i];

  1000 		int want_escape = 0;

  1001 

  1002 		if (c == '\0')

  1003 			break;

  1004 

  1005 		if (c == dialect->delimiter ||

  1006 		    c == dialect->escapechar ||

  1007 		    c == dialect->quotechar ||

  1008 		    strchr(lineterm, c)) {

  1009 			if (dialect->quoting == QUOTE_NONE)

  1010 				want_escape = 1;

  1011 			else {

  1012 				if (c == dialect->quotechar) {

  1013 					if (dialect->doublequote)

  1014 						ADDCH(dialect->quotechar);

  1015 					else

  1016 						want_escape = 1;

  1017 				}

  1018 				if (!want_escape)

  1019 					*quoted = 1;

  1020 			}

  1021 			if (want_escape) {

  1022 				if (!dialect->escapechar) {

  1023 					PyErr_Format(error_obj, 

  1024 						     "need to escape, but no escapechar set");

  1025 					return -1;

  1026 				}

  1027 				ADDCH(dialect->escapechar);

  1028 			}

  1029 		}

  1030 		/* Copy field character into record buffer.

  1031 		 */

  1032 		ADDCH(c);

  1033 	}

  1034 

  1035 	/* If field is empty check if it needs to be quoted.

  1036 	 */

  1037 	if (i == 0 && quote_empty) {

  1038 		if (dialect->quoting == QUOTE_NONE) {

  1039 			PyErr_Format(error_obj,

  1040                                      "single empty field record must be quoted");

  1041 			return -1;

  1042 		}

  1043 		else

  1044 			*quoted = 1;

  1045 	}

  1046 

  1047 	if (*quoted) {

  1048 		if (copy_phase)

  1049 			ADDCH(dialect->quotechar);

  1050 		else

  1051 			rec_len += 2;

  1052 	}

  1053 	return rec_len;

  1054 #undef ADDCH

  1055 }

  1056 

  1057 static int

  1058 join_check_rec_size(WriterObj *self, int rec_len)

  1059 {

  1060 

  1061 	if (rec_len < 0 || rec_len > INT_MAX - MEM_INCR) {

  1062 		PyErr_NoMemory();

  1063 		return 0;

  1064 	}

  1065 

  1066 	if (rec_len > self->rec_size) {

  1067 		if (self->rec_size == 0) {

  1068 			self->rec_size = (rec_len / MEM_INCR + 1) * MEM_INCR;

  1069 			if (self->rec != NULL)

  1070 				PyMem_Free(self->rec);

  1071 			self->rec = PyMem_Malloc(self->rec_size);

  1072 		}

  1073 		else {

  1074 			char *old_rec = self->rec;

  1075 

  1076 			self->rec_size = (rec_len / MEM_INCR + 1) * MEM_INCR;

  1077 			self->rec = PyMem_Realloc(self->rec, self->rec_size);

  1078 			if (self->rec == NULL)

  1079 				PyMem_Free(old_rec);

  1080 		}

  1081 		if (self->rec == NULL) {

  1082 			PyErr_NoMemory();

  1083 			return 0;

  1084 		}

  1085 	}

  1086 	return 1;

  1087 }

  1088 

  1089 static int

  1090 join_append(WriterObj *self, char *field, int *quoted, int quote_empty)

  1091 {

  1092 	int rec_len;

  1093 

  1094 	rec_len = join_append_data(self, field, quote_empty, quoted, 0);

  1095 	if (rec_len < 0)

  1096 		return 0;

  1097 

  1098 	/* grow record buffer if necessary */

  1099 	if (!join_check_rec_size(self, rec_len))

  1100 		return 0;

  1101 

  1102 	self->rec_len = join_append_data(self, field, quote_empty, quoted, 1);

  1103 	self->num_fields++;

  1104 

  1105 	return 1;

  1106 }

  1107 

  1108 static int

  1109 join_append_lineterminator(WriterObj *self)

  1110 {

  1111 	int terminator_len;

  1112 	char *terminator;

  1113 

  1114 	terminator_len = PyString_Size(self->dialect->lineterminator);

  1115 	if (terminator_len == -1)

  1116 		return 0;

  1117 

  1118 	/* grow record buffer if necessary */

  1119 	if (!join_check_rec_size(self, self->rec_len + terminator_len))

  1120 		return 0;

  1121 

  1122 	terminator = PyString_AsString(self->dialect->lineterminator); 

  1123 	if (terminator == NULL)

  1124 		return 0;

  1125 	memmove(self->rec + self->rec_len, terminator, terminator_len);

  1126 	self->rec_len += terminator_len;

  1127 

  1128 	return 1;

  1129 }

  1130 

  1131 PyDoc_STRVAR(csv_writerow_doc,

  1132 "writerow(sequence)\n"

  1133 "\n"

  1134 "Construct and write a CSV record from a sequence of fields.  Non-string\n"

  1135 "elements will be converted to string.");