PostGIS  2.1.10dev-r@@SVN_REVISION@@
shp2pgsql-core.c
Go to the documentation of this file.
1 /**********************************************************************
2  *
3  * PostGIS - Spatial Types for PostgreSQL
4  * http://www.postgis.org
5  *
6  * Copyright (C) 2008 OpenGeo.org
7  * Copyright (C) 2009 Mark Cave-Ayland <mark.cave-ayland@siriusit.co.uk>
8  *
9  * This is free software; you can redistribute and/or modify it under
10  * the terms of the GNU General Public Licence. See the COPYING file.
11  *
12  * Maintainer: Paul Ramsey <pramsey@opengeo.org>
13  *
14  **********************************************************************/
15 
16 #include "../postgis_config.h"
17 
18 #include "shp2pgsql-core.h"
19 #include "../liblwgeom/liblwgeom.h" /* for lw_vasprintf */
20 #include "../liblwgeom/lwgeom_log.h" /* for LWDEBUG macros */
21 
22 
23 /* Internal ring/point structures */
24 typedef struct struct_point
25 {
26  double x, y, z, m;
27 } Point;
28 
29 typedef struct struct_ring
30 {
31  Point *list; /* list of points */
32  struct struct_ring *next;
33  int n; /* number of points in list */
34  unsigned int linked; /* number of "next" rings */
35 } Ring;
36 
37 
38 /*
39  * Internal functions
40  */
41 
42 #define UTF8_GOOD_RESULT 0
43 #define UTF8_BAD_RESULT 1
44 #define UTF8_NO_RESULT 2
45 
46 int utf8(const char *fromcode, char *inputbuf, char **outputbuf);
47 char *escape_copy_string(char *str);
48 char *escape_insert_string(char *str);
49 
50 int GeneratePointGeometry(SHPLOADERSTATE *state, SHPObject *obj, char **geometry, int force_multi);
51 int GenerateLineStringGeometry(SHPLOADERSTATE *state, SHPObject *obj, char **geometry);
52 int PIP(Point P, Point *V, int n);
53 int FindPolygons(SHPObject *obj, Ring ***Out);
54 void ReleasePolygons(Ring **polys, int npolys);
55 int GeneratePolygonGeometry(SHPLOADERSTATE *state, SHPObject *obj, char **geometry);
56 
57 /* Append variadic formatted string to a stringbuffer */
58 void
59 vasbappend(stringbuffer_t *sb, char *fmt, ... )
60 {
61  va_list ap;
62  char *msg;
63 
64  va_start(ap, fmt);
65 
66  if (!lw_vasprintf (&msg, fmt, ap))
67  {
68  va_end (ap);
69  return;
70  }
71 
72  /* Append to the stringbuffer */
73  stringbuffer_append(sb, msg);
74  free(msg);
75 
76  va_end(ap);
77 }
78 
79 /* Return allocated string containing UTF8 string converted from encoding fromcode */
80 int utf8(const char *fromcode, char *inputbuf, char **outputbuf)
81 {
82  iconv_t cd;
83  char *outputptr;
84  size_t outbytesleft;
85  size_t inbytesleft;
86 
87  inbytesleft = strlen(inputbuf);
88 
89  cd = iconv_open("UTF-8", fromcode);
90  if ( cd == ((iconv_t)(-1)) )
91  return UTF8_NO_RESULT;
92 
93  outbytesleft = inbytesleft * 3 + 1; /* UTF8 string can be 3 times larger */
94  /* then local string */
95  *outputbuf = (char *)malloc(outbytesleft);
96  if (!*outputbuf)
97  return UTF8_NO_RESULT;
98 
99  memset(*outputbuf, 0, outbytesleft);
100  outputptr = *outputbuf;
101 
102  /* Does this string convert cleanly? */
103  if ( iconv(cd, &inputbuf, &inbytesleft, &outputptr, &outbytesleft) == -1 )
104  {
105 #ifdef HAVE_ICONVCTL
106  int on = 1;
107  /* No. Try to convert it while transliterating. */
108  iconvctl(cd, ICONV_SET_TRANSLITERATE, &on);
109  if ( iconv(cd, &inputbuf, &inbytesleft, &outputptr, &outbytesleft) == -1 )
110  {
111  /* No. Try to convert it while discarding errors. */
112  iconvctl(cd, ICONV_SET_DISCARD_ILSEQ, &on);
113  if ( iconv(cd, &inputbuf, &inbytesleft, &outputptr, &outbytesleft) == -1 )
114  {
115  /* Still no. Throw away the buffer and return. */
116  free(*outputbuf);
117  iconv_close(cd);
118  return UTF8_NO_RESULT;
119  }
120  }
121  iconv_close(cd);
122  return UTF8_BAD_RESULT;
123 #else
124  free(*outputbuf);
125  iconv_close(cd);
126  return UTF8_NO_RESULT;
127 #endif
128  }
129  /* Return a good result, converted string is in buffer. */
130  iconv_close(cd);
131  return UTF8_GOOD_RESULT;
132 }
133 
138 char *
140 {
141  /*
142  * Escape the following characters by adding a preceding backslash
143  * tab, backslash, cr, lf
144  *
145  * 1. find # of escaped characters
146  * 2. make new string
147  *
148  */
149 
150  char *result;
151  char *ptr, *optr;
152  int toescape = 0;
153  size_t size;
154 
155  ptr = str;
156 
157  /* Count how many characters we need to escape so we know the size of the string we need to return */
158  while (*ptr)
159  {
160  if (*ptr == '\t' || *ptr == '\\' || *ptr == '\n' || *ptr == '\r')
161  toescape++;
162 
163  ptr++;
164  }
165 
166  /* If we don't have to escape anything, simply return the input pointer */
167  if (toescape == 0)
168  return str;
169 
170  size = ptr - str + toescape + 1;
171  result = calloc(1, size);
172  optr = result;
173  ptr = str;
174 
175  while (*ptr)
176  {
177  if ( *ptr == '\t' || *ptr == '\\' || *ptr == '\n' || *ptr == '\r' )
178  *optr++ = '\\';
179 
180  *optr++ = *ptr++;
181  }
182 
183  *optr = '\0';
184 
185  return result;
186 }
187 
188 
193 char *
195 {
196  /*
197  * Escape single quotes by adding a preceding single quote
198  *
199  * 1. find # of characters
200  * 2. make new string
201  */
202 
203  char *result;
204  char *ptr, *optr;
205  int toescape = 0;
206  size_t size;
207 
208  ptr = str;
209 
210  /* Count how many characters we need to escape so we know the size of the string we need to return */
211  while (*ptr)
212  {
213  if (*ptr == '\'')
214  toescape++;
215 
216  ptr++;
217  }
218 
219  /* If we don't have to escape anything, simply return the input pointer */
220  if (toescape == 0)
221  return str;
222 
223  size = ptr - str + toescape + 1;
224  result = calloc(1, size);
225  optr = result;
226  ptr = str;
227 
228  while (*ptr)
229  {
230  if (*ptr == '\'')
231  *optr++='\'';
232 
233  *optr++ = *ptr++;
234  }
235 
236  *optr='\0';
237 
238  return result;
239 }
240 
241 
246 int
247 GeneratePointGeometry(SHPLOADERSTATE *state, SHPObject *obj, char **geometry, int force_multi)
248 {
249  LWGEOM **lwmultipoints;
250  LWGEOM *lwgeom = NULL;
251 
252  POINT4D point4d;
253 
254  int dims = 0;
255  int u;
256 
257  char *mem;
258  size_t mem_length;
259 
260  FLAGS_SET_Z(dims, state->has_z);
261  FLAGS_SET_M(dims, state->has_m);
262 
263  /* Allocate memory for our array of LWPOINTs and our dynptarrays */
264  lwmultipoints = malloc(sizeof(LWPOINT *) * obj->nVertices);
265 
266  /* We need an array of pointers to each of our sub-geometries */
267  for (u = 0; u < obj->nVertices; u++)
268  {
269  /* Create a ptarray containing a single point */
270  POINTARRAY *pa = ptarray_construct_empty(state->has_z, state->has_m, 1);
271 
272  /* Generate the point */
273  point4d.x = obj->padfX[u];
274  point4d.y = obj->padfY[u];
275 
276  if (state->has_z)
277  point4d.z = obj->padfZ[u];
278  if (state->has_m)
279  point4d.m = obj->padfM[u];
280 
281  /* Add in the point! */
282  ptarray_append_point(pa, &point4d, LW_TRUE);
283 
284  /* Generate the LWPOINT */
285  lwmultipoints[u] = lwpoint_as_lwgeom(lwpoint_construct(state->from_srid, NULL, pa));
286  }
287 
288  /* If we have more than 1 vertex then we are working on a MULTIPOINT and so generate a MULTIPOINT
289  rather than a POINT */
290  if ((obj->nVertices > 1) || force_multi)
291  {
292  lwgeom = lwcollection_as_lwgeom(lwcollection_construct(MULTIPOINTTYPE, state->from_srid, NULL, obj->nVertices, lwmultipoints));
293  }
294  else
295  {
296  lwgeom = lwmultipoints[0];
297  lwfree(lwmultipoints);
298  }
299 
300  if (state->config->use_wkt)
301  {
302  mem = lwgeom_to_wkt(lwgeom, WKT_EXTENDED, WKT_PRECISION, &mem_length);
303  }
304  else
305  {
306  mem = lwgeom_to_hexwkb(lwgeom, WKB_EXTENDED, &mem_length);
307  }
308 
309  if ( !mem )
310  {
311  snprintf(state->message, SHPLOADERMSGLEN, "unable to write geometry");
312  return SHPLOADERERR;
313  }
314 
315  /* Free all of the allocated items */
316  lwgeom_free(lwgeom);
317 
318  /* Return the string - everything ok */
319  *geometry = mem;
320 
321  return SHPLOADEROK;
322 }
323 
324 
328 int
330 {
331 
332  LWGEOM **lwmultilinestrings;
333  LWGEOM *lwgeom = NULL;
334  POINT4D point4d;
335  int dims = 0;
336  int u, v, start_vertex, end_vertex;
337  char *mem;
338  size_t mem_length;
339 
340 
341  FLAGS_SET_Z(dims, state->has_z);
342  FLAGS_SET_M(dims, state->has_m);
343 
344  if (state->config->simple_geometries == 1 && obj->nParts > 1)
345  {
346  snprintf(state->message, SHPLOADERMSGLEN, _("We have a Multilinestring with %d parts, can't use -S switch!"), obj->nParts);
347 
348  return SHPLOADERERR;
349  }
350 
351  /* Allocate memory for our array of LWLINEs and our dynptarrays */
352  lwmultilinestrings = malloc(sizeof(LWPOINT *) * obj->nParts);
353 
354  /* We need an array of pointers to each of our sub-geometries */
355  for (u = 0; u < obj->nParts; u++)
356  {
357  /* Create a ptarray containing the line points */
358  POINTARRAY *pa = ptarray_construct_empty(state->has_z, state->has_m, obj->nParts);
359 
360  /* Set the start/end vertices depending upon whether this is
361  a MULTILINESTRING or not */
362  if ( u == obj->nParts-1 )
363  end_vertex = obj->nVertices;
364  else
365  end_vertex = obj->panPartStart[u + 1];
366 
367  start_vertex = obj->panPartStart[u];
368 
369  for (v = start_vertex; v < end_vertex; v++)
370  {
371  /* Generate the point */
372  point4d.x = obj->padfX[v];
373  point4d.y = obj->padfY[v];
374 
375  if (state->has_z)
376  point4d.z = obj->padfZ[v];
377  if (state->has_m)
378  point4d.m = obj->padfM[v];
379 
380  ptarray_append_point(pa, &point4d, LW_FALSE);
381  }
382 
383  /* Generate the LWLINE */
384  lwmultilinestrings[u] = lwline_as_lwgeom(lwline_construct(state->from_srid, NULL, pa));
385  }
386 
387  /* If using MULTILINESTRINGs then generate the serialized collection, otherwise just a single LINESTRING */
388  if (state->config->simple_geometries == 0)
389  {
390  lwgeom = lwcollection_as_lwgeom(lwcollection_construct(MULTILINETYPE, state->from_srid, NULL, obj->nParts, lwmultilinestrings));
391  }
392  else
393  {
394  lwgeom = lwmultilinestrings[0];
395  lwfree(lwmultilinestrings);
396  }
397 
398  if (!state->config->use_wkt)
399  mem = lwgeom_to_hexwkb(lwgeom, WKB_EXTENDED, &mem_length);
400  else
401  mem = lwgeom_to_wkt(lwgeom, WKT_EXTENDED, WKT_PRECISION, &mem_length);
402 
403  if ( !mem )
404  {
405  snprintf(state->message, SHPLOADERMSGLEN, "unable to write geometry");
406  return SHPLOADERERR;
407  }
408 
409  /* Free all of the allocated items */
410  lwgeom_free(lwgeom);
411 
412  /* Return the string - everything ok */
413  *geometry = mem;
414 
415  return SHPLOADEROK;
416 }
417 
418 
425 int
426 PIP(Point P, Point *V, int n)
427 {
428  int cn = 0; /* the crossing number counter */
429  int i;
430 
431  /* loop through all edges of the polygon */
432  for (i = 0; i < n-1; i++) /* edge from V[i] to V[i+1] */
433  {
434  if (((V[i].y <= P.y) && (V[i + 1].y > P.y)) /* an upward crossing */
435  || ((V[i].y > P.y) && (V[i + 1].y <= P.y))) /* a downward crossing */
436  {
437  double vt = (float)(P.y - V[i].y) / (V[i + 1].y - V[i].y);
438  if (P.x < V[i].x + vt * (V[i + 1].x - V[i].x)) /* P.x < intersect */
439  ++cn; /* a valid crossing of y=P.y right of P.x */
440  }
441  }
442 
443  return (cn&1); /* 0 if even (out), and 1 if odd (in) */
444 }
445 
446 
447 int
449 {
450  Ring **Outer; /* Pointers to Outer rings */
451  int out_index=0; /* Count of Outer rings */
452  Ring **Inner; /* Pointers to Inner rings */
453  int in_index=0; /* Count of Inner rings */
454  int pi; /* part index */
455 
456 #if POSTGIS_DEBUG_LEVEL > 0
457  static int call = -1;
458  call++;
459 #endif
460 
461  LWDEBUGF(4, "FindPolygons[%d]: allocated space for %d rings\n", call, obj->nParts);
462 
463  /* Allocate initial memory */
464  Outer = (Ring **)malloc(sizeof(Ring *) * obj->nParts);
465  Inner = (Ring **)malloc(sizeof(Ring *) * obj->nParts);
466 
467  /* Iterate over rings dividing in Outers and Inners */
468  for (pi=0; pi < obj->nParts; pi++)
469  {
470  int vi; /* vertex index */
471  int vs; /* start index */
472  int ve; /* end index */
473  int nv; /* number of vertex */
474  double area = 0.0;
475  Ring *ring;
476 
477  /* Set start and end vertexes */
478  if (pi == obj->nParts - 1)
479  ve = obj->nVertices;
480  else
481  ve = obj->panPartStart[pi + 1];
482 
483  vs = obj->panPartStart[pi];
484 
485  /* Compute number of vertexes */
486  nv = ve - vs;
487 
488  /* Allocate memory for a ring */
489  ring = (Ring *)malloc(sizeof(Ring));
490  ring->list = (Point *)malloc(sizeof(Point) * nv);
491  ring->n = nv;
492  ring->next = NULL;
493  ring->linked = 0;
494 
495  /* Iterate over ring vertexes */
496  for (vi = vs; vi < ve; vi++)
497  {
498  int vn = vi+1; /* next vertex for area */
499  if (vn == ve)
500  vn = vs;
501 
502  ring->list[vi - vs].x = obj->padfX[vi];
503  ring->list[vi - vs].y = obj->padfY[vi];
504  ring->list[vi - vs].z = obj->padfZ[vi];
505  ring->list[vi - vs].m = obj->padfM[vi];
506 
507  area += (obj->padfX[vi] * obj->padfY[vn]) -
508  (obj->padfY[vi] * obj->padfX[vn]);
509  }
510 
511  /* Close the ring with first vertex */
512  /*ring->list[vi].x = obj->padfX[vs]; */
513  /*ring->list[vi].y = obj->padfY[vs]; */
514  /*ring->list[vi].z = obj->padfZ[vs]; */
515  /*ring->list[vi].m = obj->padfM[vs]; */
516 
517  /* Clockwise (or single-part). It's an Outer Ring ! */
518  if (area < 0.0 || obj->nParts == 1)
519  {
520  Outer[out_index] = ring;
521  out_index++;
522  }
523  else
524  {
525  /* Counterclockwise. It's an Inner Ring ! */
526  Inner[in_index] = ring;
527  in_index++;
528  }
529  }
530 
531  LWDEBUGF(4, "FindPolygons[%d]: found %d Outer, %d Inners\n", call, out_index, in_index);
532 
533  /* Put the inner rings into the list of the outer rings */
534  /* of which they are within */
535  for (pi = 0; pi < in_index; pi++)
536  {
537  Point pt, pt2;
538  int i;
539  Ring *inner = Inner[pi], *outer = NULL;
540 
541  pt.x = inner->list[0].x;
542  pt.y = inner->list[0].y;
543 
544  pt2.x = inner->list[1].x;
545  pt2.y = inner->list[1].y;
546 
547  /*
548  * If we assume that the case of the "big polygon w/o hole
549  * containing little polygon w/ hold" is ordered so that the
550  * big polygon comes first, then checking the list in reverse
551  * will assign the little polygon's hole to the little polygon
552  * w/o a lot of extra fancy containment logic here
553  */
554  for (i = out_index - 1; i >= 0; i--)
555  {
556  int in;
557 
558  in = PIP(pt, Outer[i]->list, Outer[i]->n);
559  if ( in || PIP(pt2, Outer[i]->list, Outer[i]->n) )
560  {
561  outer = Outer[i];
562  break;
563  }
564  }
565 
566  if (outer)
567  {
568  outer->linked++;
569  while (outer->next)
570  outer = outer->next;
571 
572  outer->next = inner;
573  }
574  else
575  {
576  /* The ring wasn't within any outer rings, */
577  /* assume it is a new outer ring. */
578  LWDEBUGF(4, "FindPolygons[%d]: hole %d is orphan\n", call, pi);
579 
580  Outer[out_index] = inner;
581  out_index++;
582  }
583  }
584 
585  *Out = Outer;
586  /*
587  * Only free the containing Inner array, not the ring elements, because
588  * the rings are now owned by the linked lists in the Outer array elements.
589  */
590  free(Inner);
591 
592  return out_index;
593 }
594 
595 
596 void
597 ReleasePolygons(Ring **polys, int npolys)
598 {
599  int pi;
600 
601  /* Release all memory */
602  for (pi = 0; pi < npolys; pi++)
603  {
604  Ring *Poly, *temp;
605  Poly = polys[pi];
606  while (Poly != NULL)
607  {
608  temp = Poly;
609  Poly = Poly->next;
610  free(temp->list);
611  free(temp);
612  }
613  }
614 
615  free(polys);
616 }
617 
618 
626 int
627 GeneratePolygonGeometry(SHPLOADERSTATE *state, SHPObject *obj, char **geometry)
628 {
629  Ring **Outer;
630  int polygon_total, ring_total;
631  int pi, vi; /* part index and vertex index */
632 
633  LWGEOM **lwpolygons;
634  LWGEOM *lwgeom;
635 
636  POINT4D point4d;
637 
638  int dims = 0;
639 
640  char *mem;
641  size_t mem_length;
642 
643  FLAGS_SET_Z(dims, state->has_z);
644  FLAGS_SET_M(dims, state->has_m);
645 
646  polygon_total = FindPolygons(obj, &Outer);
647 
648  if (state->config->simple_geometries == 1 && polygon_total != 1) /* We write Non-MULTI geometries, but have several parts: */
649  {
650  snprintf(state->message, SHPLOADERMSGLEN, _("We have a Multipolygon with %d parts, can't use -S switch!"), polygon_total);
651 
652  return SHPLOADERERR;
653  }
654 
655  /* Allocate memory for our array of LWPOLYs */
656  lwpolygons = malloc(sizeof(LWPOLY *) * polygon_total);
657 
658  /* Cycle through each individual polygon */
659  for (pi = 0; pi < polygon_total; pi++)
660  {
661  LWPOLY *lwpoly = lwpoly_construct_empty(state->from_srid, state->has_z, state->has_m);
662 
663  Ring *polyring;
664  int ring_index = 0;
665 
666  /* Firstly count through the total number of rings in this polygon */
667  ring_total = 0;
668  polyring = Outer[pi];
669  while (polyring)
670  {
671  ring_total++;
672  polyring = polyring->next;
673  }
674 
675  /* Cycle through each ring within the polygon, starting with the outer */
676  polyring = Outer[pi];
677 
678  while (polyring)
679  {
680  /* Create a POINTARRAY containing the points making up the ring */
681  POINTARRAY *pa = ptarray_construct_empty(state->has_z, state->has_m, polyring->n);
682 
683  for (vi = 0; vi < polyring->n; vi++)
684  {
685  /* Build up a point array of all the points in this ring */
686  point4d.x = polyring->list[vi].x;
687  point4d.y = polyring->list[vi].y;
688 
689  if (state->has_z)
690  point4d.z = polyring->list[vi].z;
691  if (state->has_m)
692  point4d.m = polyring->list[vi].m;
693 
694  ptarray_append_point(pa, &point4d, LW_TRUE);
695  }
696 
697  /* Copy the POINTARRAY pointer so we can use the LWPOLY constructor */
698  lwpoly_add_ring(lwpoly, pa);
699 
700  polyring = polyring->next;
701  ring_index++;
702  }
703 
704  /* Generate the LWGEOM */
705  lwpolygons[pi] = lwpoly_as_lwgeom(lwpoly);
706  }
707 
708  /* If using MULTIPOLYGONS then generate the serialized collection, otherwise just a single POLYGON */
709  if (state->config->simple_geometries == 0)
710  {
711  lwgeom = lwcollection_as_lwgeom(lwcollection_construct(MULTIPOLYGONTYPE, state->from_srid, NULL, polygon_total, lwpolygons));
712  }
713  else
714  {
715  lwgeom = lwpolygons[0];
716  lwfree(lwpolygons);
717  }
718 
719  if (!state->config->use_wkt)
720  mem = lwgeom_to_hexwkb(lwgeom, WKB_EXTENDED, &mem_length);
721  else
722  mem = lwgeom_to_wkt(lwgeom, WKT_EXTENDED, WKT_PRECISION, &mem_length);
723 
724  if ( !mem )
725  {
726  /* Free the linked list of rings */
727  ReleasePolygons(Outer, polygon_total);
728 
729  snprintf(state->message, SHPLOADERMSGLEN, "unable to write geometry");
730  return SHPLOADERERR;
731  }
732 
733  /* Free all of the allocated items */
734  lwgeom_free(lwgeom);
735 
736  /* Free the linked list of rings */
737  ReleasePolygons(Outer, polygon_total);
738 
739  /* Return the string - everything ok */
740  *geometry = mem;
741 
742  return SHPLOADEROK;
743 }
744 
745 
746 /*
747  * External functions (defined in shp2pgsql-core.h)
748  */
749 
750 
751 /* Convert the string to lower case */
752 void
753 strtolower(char *s)
754 {
755  int j;
756 
757  for (j = 0; j < strlen(s); j++)
758  s[j] = tolower(s[j]);
759 }
760 
761 
762 /* Default configuration settings */
763 void
765 {
766  config->opt = 'c';
767  config->table = NULL;
768  config->schema = NULL;
769  config->geo_col = NULL;
770  config->shp_file = NULL;
771  config->dump_format = 0;
772  config->simple_geometries = 0;
773  config->geography = 0;
774  config->quoteidentifiers = 0;
775  config->forceint4 = 0;
776  config->createindex = 0;
777  config->readshape = 1;
779  config->encoding = strdup(ENCODING_DEFAULT);
781  config->sr_id = SRID_UNKNOWN;
782  config->shp_sr_id = SRID_UNKNOWN;
783  config->use_wkt = 0;
784  config->tablespace = NULL;
785  config->idxtablespace = NULL;
786  config->usetransaction = 1;
787 }
788 
789 /* Create a new shapefile state object */
792 {
793  SHPLOADERSTATE *state;
794 
795  /* Create a new state object and assign the config to it */
796  state = malloc(sizeof(SHPLOADERSTATE));
797  state->config = config;
798 
799  /* Set any state defaults */
800  state->hSHPHandle = NULL;
801  state->hDBFHandle = NULL;
802  state->has_z = 0;
803  state->has_m = 0;
804  state->types = NULL;
805  state->widths = NULL;
806  state->precisions = NULL;
807  state->col_names = NULL;
808  state->field_names = NULL;
809 
810  state->from_srid = config->shp_sr_id;
811  state->to_srid = config->sr_id;
812 
813  /* If only one has a valid SRID, use it for both. */
814  if (state->to_srid == SRID_UNKNOWN)
815  {
816  if (config->geography)
817  {
818  state->to_srid = 4326;
819  }
820  else
821  {
822  state->to_srid = state->from_srid;
823  }
824  }
825 
826  if (state->from_srid == SRID_UNKNOWN)
827  {
828  state->from_srid = state->to_srid;
829  }
830 
831  /* If the geo col name is not set, use one of the defaults. */
832  state->geo_col = config->geo_col;
833 
834  if (!state->geo_col)
835  {
836  state->geo_col = strdup(config->geography ? GEOGRAPHY_DEFAULT : GEOMETRY_DEFAULT);
837  }
838 
839  return state;
840 }
841 
842 
843 /* Open the shapefile and extract the relevant field information */
844 int
846 {
847  SHPObject *obj = NULL;
848  int j, z;
849  int ret = SHPLOADEROK;
850 
851  int field_precision, field_width;
852  char name[MAXFIELDNAMELEN];
853  char name2[MAXFIELDNAMELEN];
854  DBFFieldType type = -1;
855  char *utf8str;
856 
857  /* If we are reading the entire shapefile, open it */
858  if (state->config->readshape == 1)
859  {
860  state->hSHPHandle = SHPOpen(state->config->shp_file, "rb");
861 
862  if (state->hSHPHandle == NULL)
863  {
864  snprintf(state->message, SHPLOADERMSGLEN, _("%s: shape (.shp) or index files (.shx) can not be opened, will just import attribute data."), state->config->shp_file);
865  state->config->readshape = 0;
866 
867  ret = SHPLOADERWARN;
868  }
869  }
870 
871  /* Open the DBF (attributes) file */
872  state->hDBFHandle = DBFOpen(state->config->shp_file, "rb");
873  if ((state->hSHPHandle == NULL && state->config->readshape == 1) || state->hDBFHandle == NULL)
874  {
875  snprintf(state->message, SHPLOADERMSGLEN, _("%s: dbf file (.dbf) can not be opened."), state->config->shp_file);
876 
877  return SHPLOADERERR;
878  }
879 
880  /* If reading the whole shapefile (not just attributes)... */
881  if (state->config->readshape == 1)
882  {
883  SHPGetInfo(state->hSHPHandle, &state->num_entities, &state->shpfiletype, NULL, NULL);
884 
885  /* If null_policy is set to abort, check for NULLs */
886  if (state->config->null_policy == POLICY_NULL_ABORT)
887  {
888  /* If we abort on null items, scan the entire file for NULLs */
889  for (j = 0; j < state->num_entities; j++)
890  {
891  obj = SHPReadObject(state->hSHPHandle, j);
892 
893  if (!obj)
894  {
895  snprintf(state->message, SHPLOADERMSGLEN, _("Error reading shape object %d"), j);
896  return SHPLOADERERR;
897  }
898 
899  if (obj->nVertices == 0)
900  {
901  snprintf(state->message, SHPLOADERMSGLEN, _("Empty geometries found, aborted.)"));
902  return SHPLOADERERR;
903  }
904 
905  SHPDestroyObject(obj);
906  }
907  }
908 
909  /* Check the shapefile type */
910  int geomtype = 0;
911  switch (state->shpfiletype)
912  {
913  case SHPT_POINT:
914  /* Point */
915  state->pgtype = "POINT";
916  geomtype = POINTTYPE;
917  state->pgdims = 2;
918  break;
919 
920  case SHPT_ARC:
921  /* PolyLine */
922  state->pgtype = "MULTILINESTRING";
923  geomtype = MULTILINETYPE ;
924  state->pgdims = 2;
925  break;
926 
927  case SHPT_POLYGON:
928  /* Polygon */
929  state->pgtype = "MULTIPOLYGON";
930  geomtype = MULTIPOLYGONTYPE;
931  state->pgdims = 2;
932  break;
933 
934  case SHPT_MULTIPOINT:
935  /* MultiPoint */
936  state->pgtype = "MULTIPOINT";
937  geomtype = MULTIPOINTTYPE;
938  state->pgdims = 2;
939  break;
940 
941  case SHPT_POINTM:
942  /* PointM */
943  geomtype = POINTTYPE;
944  state->has_m = 1;
945  state->pgtype = "POINTM";
946  state->pgdims = 3;
947  break;
948 
949  case SHPT_ARCM:
950  /* PolyLineM */
951  geomtype = MULTILINETYPE;
952  state->has_m = 1;
953  state->pgtype = "MULTILINESTRINGM";
954  state->pgdims = 3;
955  break;
956 
957  case SHPT_POLYGONM:
958  /* PolygonM */
959  geomtype = MULTIPOLYGONTYPE;
960  state->has_m = 1;
961  state->pgtype = "MULTIPOLYGONM";
962  state->pgdims = 3;
963  break;
964 
965  case SHPT_MULTIPOINTM:
966  /* MultiPointM */
967  geomtype = MULTIPOINTTYPE;
968  state->has_m = 1;
969  state->pgtype = "MULTIPOINTM";
970  state->pgdims = 3;
971  break;
972 
973  case SHPT_POINTZ:
974  /* PointZ */
975  geomtype = POINTTYPE;
976  state->has_m = 1;
977  state->has_z = 1;
978  state->pgtype = "POINT";
979  state->pgdims = 4;
980  break;
981 
982  case SHPT_ARCZ:
983  /* PolyLineZ */
984  state->pgtype = "MULTILINESTRING";
985  geomtype = MULTILINETYPE;
986  state->has_z = 1;
987  state->has_m = 1;
988  state->pgdims = 4;
989  break;
990 
991  case SHPT_POLYGONZ:
992  /* MultiPolygonZ */
993  state->pgtype = "MULTIPOLYGON";
994  geomtype = MULTIPOLYGONTYPE;
995  state->has_z = 1;
996  state->has_m = 1;
997  state->pgdims = 4;
998  break;
999 
1000  case SHPT_MULTIPOINTZ:
1001  /* MultiPointZ */
1002  state->pgtype = "MULTIPOINT";
1003  geomtype = MULTIPOINTTYPE;
1004  state->has_z = 1;
1005  state->has_m = 1;
1006  state->pgdims = 4;
1007  break;
1008 
1009  default:
1010  state->pgtype = "GEOMETRY";
1011  geomtype = COLLECTIONTYPE;
1012  state->has_z = 1;
1013  state->has_m = 1;
1014  state->pgdims = 4;
1015 
1016  snprintf(state->message, SHPLOADERMSGLEN, _("Unknown geometry type: %d\n"), state->shpfiletype);
1017  return SHPLOADERERR;
1018 
1019  break;
1020  }
1021 
1022  /* Force Z/M-handling if configured to do so */
1023  switch(state->config->force_output)
1024  {
1025  case FORCE_OUTPUT_2D:
1026  state->has_z = 0;
1027  state->has_m = 0;
1028  state->pgdims = 2;
1029  break;
1030 
1031  case FORCE_OUTPUT_3DZ:
1032  state->has_z = 1;
1033  state->has_m = 0;
1034  state->pgdims = 3;
1035  break;
1036 
1037  case FORCE_OUTPUT_3DM:
1038  state->has_z = 0;
1039  state->has_m = 1;
1040  state->pgdims = 3;
1041  break;
1042 
1043  case FORCE_OUTPUT_4D:
1044  state->has_z = 1;
1045  state->has_m = 1;
1046  state->pgdims = 4;
1047  break;
1048  default:
1049  /* Simply use the auto-detected values above */
1050  break;
1051  }
1052 
1053  /* If in simple geometry mode, alter names for CREATE TABLE by skipping MULTI */
1054  if (state->config->simple_geometries)
1055  {
1056  if ((geomtype == MULTIPOLYGONTYPE) || (geomtype == MULTILINETYPE) || (geomtype == MULTIPOINTTYPE))
1057  {
1058  /* Chop off the "MULTI" from the string. */
1059  state->pgtype += 5;
1060  }
1061  }
1062 
1063  }
1064  else
1065  {
1066  /* Otherwise just count the number of records in the DBF */
1067  state->num_entities = DBFGetRecordCount(state->hDBFHandle);
1068  }
1069 
1070 
1071  /* Get the field information from the DBF */
1072  state->num_fields = DBFGetFieldCount(state->hDBFHandle);
1073  state->num_records = DBFGetRecordCount(state->hDBFHandle);
1074 
1075  /* Allocate storage for field information */
1076  state->field_names = malloc(state->num_fields * sizeof(char*));
1077  state->types = (DBFFieldType *)malloc(state->num_fields * sizeof(int));
1078  state->widths = malloc(state->num_fields * sizeof(int));
1079  state->precisions = malloc(state->num_fields * sizeof(int));
1080  state->pgfieldtypes = malloc(state->num_fields * sizeof(char *));
1081  state->col_names = malloc((state->num_fields + 2) * sizeof(char) * MAXFIELDNAMELEN);
1082 
1083  /* Generate a string of comma separated column names of the form "(col1, col2 ... colN)" for the SQL
1084  insertion string */
1085  strcpy(state->col_names, "(" );
1086 
1087  for (j = 0; j < state->num_fields; j++)
1088  {
1089  type = DBFGetFieldInfo(state->hDBFHandle, j, name, &field_width, &field_precision);
1090 
1091  state->types[j] = type;
1092  state->widths[j] = field_width;
1093  state->precisions[j] = field_precision;
1094 
1095  if (state->config->encoding)
1096  {
1097  char *encoding_msg = _("Try \"LATIN1\" (Western European), or one of the values described at http://www.gnu.org/software/libiconv/.");
1098 
1099  int rv = utf8(state->config->encoding, name, &utf8str);
1100 
1101  if (rv != UTF8_GOOD_RESULT)
1102  {
1103  if ( rv == UTF8_BAD_RESULT )
1104  snprintf(state->message, SHPLOADERMSGLEN, _("Unable to convert field name \"%s\" to UTF-8 (iconv reports \"%s\"). Current encoding is \"%s\". %s"), utf8str, strerror(errno), state->config->encoding, encoding_msg);
1105  else if ( rv == UTF8_NO_RESULT )
1106  snprintf(state->message, SHPLOADERMSGLEN, _("Unable to convert field name to UTF-8 (iconv reports \"%s\"). Current encoding is \"%s\". %s"), strerror(errno), state->config->encoding, encoding_msg);
1107  else
1108  snprintf(state->message, SHPLOADERMSGLEN, _("Unexpected return value from utf8()"));
1109 
1110  if ( rv == UTF8_BAD_RESULT )
1111  free(utf8str);
1112 
1113  return SHPLOADERERR;
1114  }
1115 
1116  strncpy(name, utf8str, MAXFIELDNAMELEN);
1117  free(utf8str);
1118  }
1119 
1120  /*
1121  * Make field names lowercase unless asked to
1122  * keep identifiers case.
1123  */
1124  if (!state->config->quoteidentifiers)
1125  strtolower(name);
1126 
1127  /*
1128  * Escape names starting with the
1129  * escape char (_), those named 'gid'
1130  * or after pgsql reserved attribute names
1131  */
1132  if (name[0] == '_' ||
1133  ! strcmp(name, "gid") || ! strcmp(name, "tableoid") ||
1134  ! strcmp(name, "cmin") ||
1135  ! strcmp(name, "cmax") ||
1136  ! strcmp(name, "xmin") ||
1137  ! strcmp(name, "xmax") ||
1138  ! strcmp(name, "primary") ||
1139  ! strcmp(name, "oid") || ! strcmp(name, "ctid"))
1140  {
1141  strncpy(name2 + 2, name, MAXFIELDNAMELEN - 2);
1142  name2[0] = '_';
1143  name2[1] = '_';
1144  strcpy(name, name2);
1145  }
1146 
1147  /* Avoid duplicating field names */
1148  for (z = 0; z < j ; z++)
1149  {
1150  if (strcmp(state->field_names[z], name) == 0)
1151  {
1152  strncat(name, "__", MAXFIELDNAMELEN);
1153  snprintf(name + strlen(name), MAXFIELDNAMELEN, "%i", j);
1154  break;
1155  }
1156  }
1157 
1158  state->field_names[j] = malloc(strlen(name) + 1);
1159  strcpy(state->field_names[j], name);
1160 
1161  /* Now generate the PostgreSQL type name string and width based upon the shapefile type */
1162  switch (state->types[j])
1163  {
1164  case FTString:
1165  state->pgfieldtypes[j] = malloc(strlen("varchar") + 1);
1166  strcpy(state->pgfieldtypes[j], "varchar");
1167  break;
1168 
1169  case FTDate:
1170  state->pgfieldtypes[j] = malloc(strlen("date") + 1);
1171  strcpy(state->pgfieldtypes[j], "date");
1172  break;
1173 
1174  case FTInteger:
1175  /* Determine exact type based upon field width */
1176  if (state->config->forceint4 || (state->widths[j] >=5 && state->widths[j] < 10))
1177  {
1178  state->pgfieldtypes[j] = malloc(strlen("int4") + 1);
1179  strcpy(state->pgfieldtypes[j], "int4");
1180  }
1181  else if (state->widths[j] < 5)
1182  {
1183  state->pgfieldtypes[j] = malloc(strlen("int2") + 1);
1184  strcpy(state->pgfieldtypes[j], "int2");
1185  }
1186  else
1187  {
1188  state->pgfieldtypes[j] = malloc(strlen("numeric") + 1);
1189  strcpy(state->pgfieldtypes[j], "numeric");
1190  }
1191  break;
1192 
1193  case FTDouble:
1194  /* Determine exact type based upon field width */
1195  if (state->widths[j] > 18)
1196  {
1197  state->pgfieldtypes[j] = malloc(strlen("numeric") + 1);
1198  strcpy(state->pgfieldtypes[j], "numeric");
1199  }
1200  else
1201  {
1202  state->pgfieldtypes[j] = malloc(strlen("float8") + 1);
1203  strcpy(state->pgfieldtypes[j], "float8");
1204  }
1205  break;
1206 
1207  case FTLogical:
1208  state->pgfieldtypes[j] = malloc(strlen("boolean") + 1);
1209  strcpy(state->pgfieldtypes[j], "boolean");
1210  break;
1211 
1212  default:
1213  snprintf(state->message, SHPLOADERMSGLEN, _("Invalid type %x in DBF file"), state->types[j]);
1214  return SHPLOADERERR;
1215  }
1216 
1217  strcat(state->col_names, "\"");
1218  strcat(state->col_names, name);
1219 
1220  if (state->config->readshape == 1 || j < (state->num_fields - 1))
1221  {
1222  /* Don't include last comma if its the last field and no geometry field will follow */
1223  strcat(state->col_names, "\",");
1224  }
1225  else
1226  {
1227  strcat(state->col_names, "\"");
1228  }
1229  }
1230 
1231  /* Append the geometry column if required */
1232  if (state->config->readshape == 1)
1233  strcat(state->col_names, state->geo_col);
1234 
1235  strcat(state->col_names, ")");
1236 
1237 
1238  /* Return status */
1239  return ret;
1240 }
1241 
1242 /* Return a pointer to an allocated string containing the header for the specified loader state */
1243 int
1244 ShpLoaderGetSQLHeader(SHPLOADERSTATE *state, char **strheader)
1245 {
1246  stringbuffer_t *sb;
1247  char *ret;
1248  int j;
1249 
1250  /* Create the stringbuffer containing the header; we use this API as it's easier
1251  for handling string resizing during append */
1252  sb = stringbuffer_create();
1253  stringbuffer_clear(sb);
1254 
1255  /* Set the client encoding if required */
1256  if (state->config->encoding)
1257  {
1258  stringbuffer_aprintf(sb, "SET CLIENT_ENCODING TO UTF8;\n");
1259  }
1260 
1261  /* Use SQL-standard string escaping rather than PostgreSQL standard */
1262  stringbuffer_aprintf(sb, "SET STANDARD_CONFORMING_STRINGS TO ON;\n");
1263 
1264  /* Drop table if requested */
1265  if (state->config->opt == 'd')
1266  {
1278  if (state->config->schema)
1279  {
1280  if (state->config->readshape == 1 && (! state->config->geography) )
1281  {
1282  stringbuffer_aprintf(sb, "SELECT DropGeometryColumn('%s','%s','%s');\n",
1283  state->config->schema, state->config->table, state->geo_col);
1284  }
1285 
1286  stringbuffer_aprintf(sb, "DROP TABLE \"%s\".\"%s\";\n", state->config->schema,
1287  state->config->table);
1288  }
1289  else
1290  {
1291  if (state->config->readshape == 1 && (! state->config->geography) )
1292  {
1293  stringbuffer_aprintf(sb, "SELECT DropGeometryColumn('','%s','%s');\n",
1294  state->config->table, state->geo_col);
1295  }
1296 
1297  stringbuffer_aprintf(sb, "DROP TABLE \"%s\";\n", state->config->table);
1298  }
1299  }
1300 
1301  /* Start of transaction if we are using one */
1302  if (state->config->usetransaction)
1303  {
1304  stringbuffer_aprintf(sb, "BEGIN;\n");
1305  }
1306 
1307  /* If not in 'append' mode create the spatial table */
1308  if (state->config->opt != 'a')
1309  {
1310  /*
1311  * Create a table for inserting the shapes into with appropriate
1312  * columns and types
1313  */
1314  if (state->config->schema)
1315  {
1316  stringbuffer_aprintf(sb, "CREATE TABLE \"%s\".\"%s\" (gid serial",
1317  state->config->schema, state->config->table);
1318  }
1319  else
1320  {
1321  stringbuffer_aprintf(sb, "CREATE TABLE \"%s\" (gid serial", state->config->table);
1322  }
1323 
1324  /* Generate the field types based upon the shapefile information */
1325  for (j = 0; j < state->num_fields; j++)
1326  {
1327  stringbuffer_aprintf(sb, ",\n\"%s\" ", state->field_names[j]);
1328 
1329  /* First output the raw field type string */
1330  stringbuffer_aprintf(sb, "%s", state->pgfieldtypes[j]);
1331 
1332  /* Some types do have typmods though... */
1333  if (!strcmp("varchar", state->pgfieldtypes[j]))
1334  stringbuffer_aprintf(sb, "(%d)", state->widths[j]);
1335 
1336  if (!strcmp("numeric", state->pgfieldtypes[j]))
1337  {
1338  /* Doubles we just allow PostgreSQL to auto-detect the size */
1339  if (state->types[j] != FTDouble)
1340  stringbuffer_aprintf(sb, "(%d,0)", state->widths[j]);
1341  }
1342  }
1343 
1344  /* Add the geography column directly to the table definition, we don't
1345  need to do an AddGeometryColumn() call. */
1346  if (state->config->readshape == 1 && state->config->geography)
1347  {
1348  char *dimschar;
1349 
1350  if (state->pgdims == 4)
1351  dimschar = "ZM";
1352  else
1353  dimschar = "";
1354 
1355  if (state->to_srid != SRID_UNKNOWN && state->to_srid != 4326)
1356  {
1357  snprintf(state->message, SHPLOADERMSGLEN, _("Invalid SRID for geography type: %d"), state->to_srid);
1359  return SHPLOADERERR;
1360  }
1361  stringbuffer_aprintf(sb, ",\n\"%s\" geography(%s%s,%d)", state->geo_col, state->pgtype, dimschar, 4326);
1362  }
1363  stringbuffer_aprintf(sb, ")");
1364 
1365  /* Tablespace is optional. */
1366  if (state->config->tablespace != NULL)
1367  {
1368  stringbuffer_aprintf(sb, " TABLESPACE \"%s\"", state->config->tablespace);
1369  }
1370  stringbuffer_aprintf(sb, ";\n");
1371 
1372  /* Create the primary key. This is done separately because the index for the PK needs
1373  * to be in the correct tablespace. */
1374 
1375  /* TODO: Currently PostgreSQL does not allow specifying an index to use for a PK (so you get
1376  * a default one called table_pkey) and it does not provide a way to create a PK index
1377  * in a specific tablespace. So as a hacky solution we create the PK, then move the
1378  * index to the correct tablespace. Eventually this should be:
1379  * CREATE INDEX table_pkey on table(gid) TABLESPACE tblspc;
1380  * ALTER TABLE table ADD PRIMARY KEY (gid) USING INDEX table_pkey;
1381  * A patch has apparently been submitted to PostgreSQL to enable this syntax, see this thread:
1382  * http://archives.postgresql.org/pgsql-hackers/2011-01/msg01405.php */
1383  stringbuffer_aprintf(sb, "ALTER TABLE ");
1384 
1385  /* Schema is optional, include if present. */
1386  if (state->config->schema)
1387  {
1388  stringbuffer_aprintf(sb, "\"%s\".",state->config->schema);
1389  }
1390  stringbuffer_aprintf(sb, "\"%s\" ADD PRIMARY KEY (gid);\n", state->config->table);
1391 
1392  /* Tablespace is optional for the index. */
1393  if (state->config->idxtablespace != NULL)
1394  {
1395  stringbuffer_aprintf(sb, "ALTER INDEX ");
1396  if (state->config->schema)
1397  {
1398  stringbuffer_aprintf(sb, "\"%s\".",state->config->schema);
1399  }
1400 
1401  /* WARNING: We're assuming the default "table_pkey" name for the primary
1402  * key index. PostgreSQL may use "table_pkey1" or similar in the
1403  * case of a name conflict, so you may need to edit the produced
1404  * SQL in this rare case. */
1405  stringbuffer_aprintf(sb, "\"%s_pkey\" SET TABLESPACE \"%s\";\n",
1406  state->config->table, state->config->idxtablespace);
1407  }
1408 
1409  /* Create the geometry column with an addgeometry call */
1410  if (state->config->readshape == 1 && (!state->config->geography))
1411  {
1412  /* If they didn't specify a target SRID, see if they specified a source SRID. */
1413  int srid = state->to_srid;
1414  if (state->config->schema)
1415  {
1416  stringbuffer_aprintf(sb, "SELECT AddGeometryColumn('%s','%s','%s','%d',",
1417  state->config->schema, state->config->table, state->geo_col, srid);
1418  }
1419  else
1420  {
1421  stringbuffer_aprintf(sb, "SELECT AddGeometryColumn('','%s','%s','%d',",
1422  state->config->table, state->geo_col, srid);
1423  }
1424 
1425  stringbuffer_aprintf(sb, "'%s',%d);\n", state->pgtype, state->pgdims);
1426  }
1427  }
1428 
1429  /* Copy the string buffer into a new string, destroying the string buffer */
1430  ret = (char *)malloc(strlen((char *)stringbuffer_getstring(sb)) + 1);
1431  strcpy(ret, (char *)stringbuffer_getstring(sb));
1433 
1434  *strheader = ret;
1435 
1436  return SHPLOADEROK;
1437 }
1438 
1439 
1440 /* Return an allocated string containing the copy statement for this state */
1441 int
1443 {
1444  char *copystr;
1445 
1446  /* Allocate the string for the COPY statement */
1447  if (state->config->dump_format)
1448  {
1449  if (state->config->schema)
1450  {
1451  copystr = malloc(strlen(state->config->schema) + strlen(state->config->table) +
1452  strlen(state->col_names) + 40);
1453 
1454  sprintf(copystr, "COPY \"%s\".\"%s\" %s FROM stdin;\n",
1455  state->config->schema, state->config->table, state->col_names);
1456  }
1457  else
1458  {
1459  copystr = malloc(strlen(state->config->table) + strlen(state->col_names) + 40);
1460 
1461  sprintf(copystr, "COPY \"%s\" %s FROM stdin;\n", state->config->table, state->col_names);
1462  }
1463 
1464  *strheader = copystr;
1465  return SHPLOADEROK;
1466  }
1467  else
1468  {
1469  /* Flag an error as something has gone horribly wrong */
1470  snprintf(state->message, SHPLOADERMSGLEN, _("Internal error: attempt to generate a COPY statement for data that hasn't been requested in COPY format"));
1471 
1472  return SHPLOADERERR;
1473  }
1474 }
1475 
1476 
1477 /* Return a count of the number of entities in this shapefile */
1478 int
1480 {
1481  return state->num_entities;
1482 }
1483 
1484 
1485 /* Return an allocated string representation of a specified record item */
1486 int
1487 ShpLoaderGenerateSQLRowStatement(SHPLOADERSTATE *state, int item, char **strrecord)
1488 {
1489  SHPObject *obj = NULL;
1490  stringbuffer_t *sb;
1491  stringbuffer_t *sbwarn;
1492  char val[MAXVALUELEN];
1493  char *escval;
1494  char *geometry=NULL, *ret;
1495  char *utf8str;
1496  int res, i;
1497  int rv;
1498 
1499  /* Clear the stringbuffers */
1500  sbwarn = stringbuffer_create();
1501  stringbuffer_clear(sbwarn);
1502  sb = stringbuffer_create();
1503  stringbuffer_clear(sb);
1504 
1505  /* If we are reading the DBF only and the record has been marked deleted, return deleted record status */
1506  if (state->config->readshape == 0 && DBFIsRecordDeleted(state->hDBFHandle, item))
1507  {
1508  *strrecord = NULL;
1509  return SHPLOADERRECDELETED;
1510  }
1511 
1512  /* If we are reading the shapefile, open the specified record */
1513  if (state->config->readshape == 1)
1514  {
1515  obj = SHPReadObject(state->hSHPHandle, item);
1516  if (!obj)
1517  {
1518  snprintf(state->message, SHPLOADERMSGLEN, _("Error reading shape object %d"), item);
1519  return SHPLOADERERR;
1520  }
1521 
1522  /* If we are set to skip NULLs, return a NULL record status */
1523  if (state->config->null_policy == POLICY_NULL_SKIP && obj->nVertices == 0 )
1524  {
1525  SHPDestroyObject(obj);
1526 
1527  *strrecord = NULL;
1528  return SHPLOADERRECISNULL;
1529  }
1530  }
1531 
1532  /* If not in dump format, generate the INSERT string */
1533  if (!state->config->dump_format)
1534  {
1535  if (state->config->schema)
1536  {
1537  stringbuffer_aprintf(sb, "INSERT INTO \"%s\".\"%s\" %s VALUES (", state->config->schema,
1538  state->config->table, state->col_names);
1539  }
1540  else
1541  {
1542  stringbuffer_aprintf(sb, "INSERT INTO \"%s\" %s VALUES (", state->config->table,
1543  state->col_names);
1544  }
1545  }
1546 
1547 
1548  /* Read all of the attributes from the DBF file for this item */
1549  for (i = 0; i < DBFGetFieldCount(state->hDBFHandle); i++)
1550  {
1551  /* Special case for NULL attributes */
1552  if (DBFIsAttributeNULL(state->hDBFHandle, item, i))
1553  {
1554  if (state->config->dump_format)
1555  stringbuffer_aprintf(sb, "\\N");
1556  else
1557  stringbuffer_aprintf(sb, "NULL");
1558  }
1559  else
1560  {
1561  /* Attribute NOT NULL */
1562  switch (state->types[i])
1563  {
1564  case FTInteger:
1565  case FTDouble:
1566  rv = snprintf(val, MAXVALUELEN, "%s", DBFReadStringAttribute(state->hDBFHandle, item, i));
1567  if (rv >= MAXVALUELEN || rv == -1)
1568  {
1569  stringbuffer_aprintf(sbwarn, "Warning: field %d name truncated\n", i);
1570  val[MAXVALUELEN - 1] = '\0';
1571  }
1572 
1573  /* If the value is an empty string, change to 0 */
1574  if (val[0] == '\0')
1575  {
1576  val[0] = '0';
1577  val[1] = '\0';
1578  }
1579 
1580  /* If the value ends with just ".", remove the dot */
1581  if (val[strlen(val) - 1] == '.')
1582  val[strlen(val) - 1] = '\0';
1583  break;
1584 
1585  case FTString:
1586  case FTLogical:
1587  case FTDate:
1588  rv = snprintf(val, MAXVALUELEN, "%s", DBFReadStringAttribute(state->hDBFHandle, item, i));
1589  if (rv >= MAXVALUELEN || rv == -1)
1590  {
1591  stringbuffer_aprintf(sbwarn, "Warning: field %d name truncated\n", i);
1592  val[MAXVALUELEN - 1] = '\0';
1593  }
1594  break;
1595 
1596  default:
1597  snprintf(state->message, SHPLOADERMSGLEN, _("Error: field %d has invalid or unknown field type (%d)"), i, state->types[i]);
1598 
1599  /* clean up and return err */
1600  SHPDestroyObject(obj);
1601  stringbuffer_destroy(sbwarn);
1603  return SHPLOADERERR;
1604  }
1605 
1606  if (state->config->encoding)
1607  {
1608  char *encoding_msg = _("Try \"LATIN1\" (Western European), or one of the values described at http://www.postgresql.org/docs/current/static/multibyte.html.");
1609 
1610  rv = utf8(state->config->encoding, val, &utf8str);
1611 
1612  if (rv != UTF8_GOOD_RESULT)
1613  {
1614  if ( rv == UTF8_BAD_RESULT )
1615  snprintf(state->message, SHPLOADERMSGLEN, _("Unable to convert data value \"%s\" to UTF-8 (iconv reports \"%s\"). Current encoding is \"%s\". %s"), utf8str, strerror(errno), state->config->encoding, encoding_msg);
1616  else if ( rv == UTF8_NO_RESULT )
1617  snprintf(state->message, SHPLOADERMSGLEN, _("Unable to convert data value to UTF-8 (iconv reports \"%s\"). Current encoding is \"%s\". %s"), strerror(errno), state->config->encoding, encoding_msg);
1618  else
1619  snprintf(state->message, SHPLOADERMSGLEN, _("Unexpected return value from utf8()"));
1620 
1621  if ( rv == UTF8_BAD_RESULT )
1622  free(utf8str);
1623 
1624  /* clean up and return err */
1625  SHPDestroyObject(obj);
1626  stringbuffer_destroy(sbwarn);
1628  return SHPLOADERERR;
1629  }
1630  strncpy(val, utf8str, MAXVALUELEN);
1631  free(utf8str);
1632 
1633  }
1634 
1635  /* Escape attribute correctly according to dump format */
1636  if (state->config->dump_format)
1637  {
1638  escval = escape_copy_string(val);
1639  stringbuffer_aprintf(sb, "%s", escval);
1640  }
1641  else
1642  {
1643  escval = escape_insert_string(val);
1644  stringbuffer_aprintf(sb, "'%s'", escval);
1645  }
1646 
1647  /* Free the escaped version if required */
1648  if (val != escval)
1649  free(escval);
1650  }
1651 
1652  /* Only put in delimeter if not last field or a shape will follow */
1653  if (state->config->readshape == 1 || i < DBFGetFieldCount(state->hDBFHandle) - 1)
1654  {
1655  if (state->config->dump_format)
1656  stringbuffer_aprintf(sb, "\t");
1657  else
1658  stringbuffer_aprintf(sb, ",");
1659  }
1660 
1661  /* End of DBF attribute loop */
1662  }
1663 
1664 
1665  /* Add the shape attribute if we are reading it */
1666  if (state->config->readshape == 1)
1667  {
1668  /* Force the locale to C */
1669  char *oldlocale = setlocale(LC_NUMERIC, "C");
1670 
1671  /* Handle the case of a NULL shape */
1672  if (obj->nVertices == 0)
1673  {
1674  if (state->config->dump_format)
1675  stringbuffer_aprintf(sb, "\\N");
1676  else
1677  stringbuffer_aprintf(sb, "NULL");
1678  }
1679  else
1680  {
1681  /* Handle all other shape attributes */
1682  switch (obj->nSHPType)
1683  {
1684  case SHPT_POLYGON:
1685  case SHPT_POLYGONM:
1686  case SHPT_POLYGONZ:
1687  res = GeneratePolygonGeometry(state, obj, &geometry);
1688  break;
1689 
1690  case SHPT_POINT:
1691  case SHPT_POINTM:
1692  case SHPT_POINTZ:
1693  res = GeneratePointGeometry(state, obj, &geometry, 0);
1694  break;
1695 
1696  case SHPT_MULTIPOINT:
1697  case SHPT_MULTIPOINTM:
1698  case SHPT_MULTIPOINTZ:
1699  /* Force it to multi unless using -S */
1700  res = GeneratePointGeometry(state, obj, &geometry,
1701  state->config->simple_geometries ? 0 : 1);
1702  break;
1703 
1704  case SHPT_ARC:
1705  case SHPT_ARCM:
1706  case SHPT_ARCZ:
1707  res = GenerateLineStringGeometry(state, obj, &geometry);
1708  break;
1709 
1710  default:
1711  snprintf(state->message, SHPLOADERMSGLEN, _("Shape type is not supported, type id = %d"), obj->nSHPType);
1712  SHPDestroyObject(obj);
1713  stringbuffer_destroy(sbwarn);
1715 
1716  return SHPLOADERERR;
1717  }
1718  /* The default returns out of the function, so res will always have been set. */
1719  if (res != SHPLOADEROK)
1720  {
1721  /* Error message has already been set */
1722  SHPDestroyObject(obj);
1723  stringbuffer_destroy(sbwarn);
1725 
1726  return SHPLOADERERR;
1727  }
1728 
1729  /* Now generate the geometry string according to the current configuration */
1730  if (!state->config->dump_format)
1731  {
1732  if (state->to_srid != state->from_srid)
1733  {
1734  stringbuffer_aprintf(sb, "ST_Transform(");
1735  }
1736  stringbuffer_aprintf(sb, "'");
1737  }
1738 
1739  stringbuffer_aprintf(sb, "%s", geometry);
1740 
1741  if (!state->config->dump_format)
1742  {
1743  stringbuffer_aprintf(sb, "'");
1744 
1745  /* Close the ST_Transform if reprojecting. */
1746  if (state->to_srid != state->from_srid)
1747  {
1748  /* We need to add an explicit cast to geography/geometry to ensure that
1749  PostgreSQL doesn't get confused with the ST_Transform() raster
1750  function. */
1751  if (state->config->geography)
1752  stringbuffer_aprintf(sb, "::geometry, %d)::geography", state->to_srid);
1753  else
1754  stringbuffer_aprintf(sb, "::geometry, %d)", state->to_srid);
1755  }
1756  }
1757 
1758  free(geometry);
1759  }
1760 
1761  /* Tidy up everything */
1762  SHPDestroyObject(obj);
1763 
1764  setlocale(LC_NUMERIC, oldlocale);
1765  }
1766 
1767  /* Close the line correctly for dump/insert format */
1768  if (!state->config->dump_format)
1769  stringbuffer_aprintf(sb, ");");
1770 
1771 
1772  /* Copy the string buffer into a new string, destroying the string buffer */
1773  ret = (char *)malloc(strlen((char *)stringbuffer_getstring(sb)) + 1);
1774  strcpy(ret, (char *)stringbuffer_getstring(sb));
1776 
1777  *strrecord = ret;
1778 
1779  /* If any warnings occurred, set the returned message string and warning status */
1780  if (strlen((char *)stringbuffer_getstring(sbwarn)) > 0)
1781  {
1782  snprintf(state->message, SHPLOADERMSGLEN, "%s", stringbuffer_getstring(sbwarn));
1783  stringbuffer_destroy(sbwarn);
1784 
1785  return SHPLOADERWARN;
1786  }
1787  else
1788  {
1789  /* Everything went okay */
1790  stringbuffer_destroy(sbwarn);
1791 
1792  return SHPLOADEROK;
1793  }
1794 }
1795 
1796 
1797 /* Return a pointer to an allocated string containing the header for the specified loader state */
1798 int
1799 ShpLoaderGetSQLFooter(SHPLOADERSTATE *state, char **strfooter)
1800 {
1801  stringbuffer_t *sb;
1802  char *ret;
1803 
1804  /* Create the stringbuffer containing the header; we use this API as it's easier
1805  for handling string resizing during append */
1806  sb = stringbuffer_create();
1807  stringbuffer_clear(sb);
1808 
1809  /* Create gist index if specified and not in "prepare" mode */
1810  if (state->config->createindex)
1811  {
1812  stringbuffer_aprintf(sb, "CREATE INDEX ON ");
1813  /* Schema is optional, include if present. */
1814  if (state->config->schema)
1815  {
1816  stringbuffer_aprintf(sb, "\"%s\".",state->config->schema);
1817  }
1818  stringbuffer_aprintf(sb, "\"%s\" USING GIST (\"%s\")", state->config->table, state->geo_col);
1819  /* Tablespace is also optional. */
1820  if (state->config->idxtablespace != NULL)
1821  {
1822  stringbuffer_aprintf(sb, " TABLESPACE \"%s\"", state->config->idxtablespace);
1823  }
1824  stringbuffer_aprintf(sb, ";\n");
1825  }
1826 
1827  /* End the transaction if there is one. */
1828  if (state->config->usetransaction)
1829  {
1830  stringbuffer_aprintf(sb, "COMMIT;\n");
1831  }
1832 
1833  /* Copy the string buffer into a new string, destroying the string buffer */
1834  ret = (char *)malloc(strlen((char *)stringbuffer_getstring(sb)) + 1);
1835  strcpy(ret, (char *)stringbuffer_getstring(sb));
1837 
1838  *strfooter = ret;
1839 
1840  return SHPLOADEROK;
1841 }
1842 
1843 
1844 void
1846 {
1847  /* Destroy a state object created with ShpLoaderOpenShape */
1848  int i;
1849 
1850  if (state != NULL)
1851  {
1852  if (state->hSHPHandle)
1853  SHPClose(state->hSHPHandle);
1854  if (state->hDBFHandle)
1855  DBFClose(state->hDBFHandle);
1856  if (state->field_names)
1857  {
1858  for (i = 0; i < state->num_fields; i++)
1859  free(state->field_names[i]);
1860 
1861  free(state->field_names);
1862  }
1863  if (state->pgfieldtypes)
1864  {
1865  for (i = 0; i < state->num_fields; i++)
1866  free(state->pgfieldtypes[i]);
1867 
1868  free(state->pgfieldtypes);
1869  }
1870  if (state->types)
1871  free(state->types);
1872  if (state->widths)
1873  free(state->widths);
1874  if (state->precisions)
1875  free(state->precisions);
1876  if (state->col_names)
1877  free(state->col_names);
1878 
1879  /* Free the state itself */
1880  free(state);
1881  }
1882 }
double x
Definition: liblwgeom.h:308
char * lwgeom_to_hexwkb(const LWGEOM *geom, uint8_t variant, size_t *size_out)
Definition: lwout_wkb.c:776
SHPObject SHPAPI_CALL1 * SHPReadObject(SHPHandle hSHP, int iShape);int SHPAPI_CALLSHPWriteObject(SHPHandle hSHP, int iShape, SHPObject *psObject
SHPLOADERCONFIG * config
int ShpLoaderGetSQLFooter(SHPLOADERSTATE *state, char **strfooter)
#define SHPT_ARCM
Definition: shapefil.h:317
tuple res
Definition: window.py:80
DBFFieldType * types
int * panPartStart
Definition: shapefil.h:346
double m
Definition: liblwgeom.h:308
LWCOLLECTION * lwcollection_construct(uint8_t type, int srid, GBOX *bbox, uint32_t ngeoms, LWGEOM **geoms)
Definition: lwcollection.c:30
stringbuffer_t * stringbuffer_create(void)
Allocate a new stringbuffer_t.
Definition: stringbuffer.c:47
char * lwgeom_to_wkt(const LWGEOM *geom, uint8_t variant, int precision, size_t *size_out)
WKT emitter function.
Definition: lwout_wkt.c:655
void lwfree(void *mem)
Definition: lwutil.c:190
#define FORCE_OUTPUT_3DM
#define SHPT_POLYGONM
Definition: shapefil.h:318
#define FORCE_OUTPUT_3DZ
int PIP(Point P, Point *V, int n)
PIP(): crossing number test for a point in a polygon input: P = a point, V[] = vertex points of a pol...
int ShpLoaderOpenShape(SHPLOADERSTATE *state)
int nVertices
Definition: shapefil.h:349
#define POLICY_NULL_INSERT
void vasbappend(stringbuffer_t *sb, char *fmt,...)
#define SHPLOADERRECISNULL
Datum area(PG_FUNCTION_ARGS)
#define _(String)
Definition: shpcommon.h:23
#define UTF8_BAD_RESULT
const char SHPAPI_CALL1 * DBFReadStringAttribute(DBFHandle psDBF, int iRecord, int iField){return((const char *) DBFReadAttribute(psDBF, iRecord, iField, 'C')
POINTARRAY * ptarray_construct_empty(char hasz, char hasm, uint32_t maxpoints)
Create a new POINTARRAY with no points.
Definition: ptarray.c:57
void lwgeom_free(LWGEOM *geom)
Definition: lwgeom.c:1006
#define MULTIPOINTTYPE
Definition: liblwgeom.h:63
tuple fmt
Definition: pixval.py:92
#define SHPLOADERWARN
int lw_vasprintf(char **result, const char *format, va_list args)
Definition: vsprintf.c:148
double * padfX
Definition: shapefil.h:350
#define SHPT_MULTIPOINT
Definition: shapefil.h:311
#define POLICY_NULL_SKIP
#define SHPLOADERRECDELETED
#define SHPT_POLYGON
Definition: shapefil.h:310
void SHPAPI_CALL SHPDestroyObject(SHPObject *psObject)
Definition: shpopen.c:2183
#define SHPT_MULTIPOINTZ
Definition: shapefil.h:315
int GeneratePointGeometry(SHPLOADERSTATE *state, SHPObject *obj, char **geometry, int force_multi)
Generate an allocated geometry string for shapefile object obj using the state parameters if "force_m...
int GeneratePolygonGeometry(SHPLOADERSTATE *state, SHPObject *obj, char **geometry)
Generate an allocated geometry string for shapefile object obj using the state parameters.
int GenerateLineStringGeometry(SHPLOADERSTATE *state, SHPObject *obj, char **geometry)
Generate an allocated geometry string for shapefile object obj using the state parameters.
int ShpLoaderGetSQLHeader(SHPLOADERSTATE *state, char **strheader)
struct struct_ring Ring
double * padfY
Definition: shapefil.h:351
DBFHandle hDBFHandle
#define UTF8_GOOD_RESULT
char ** result
Definition: liblwgeom.h:218
LWGEOM * lwpoly_as_lwgeom(const LWPOLY *obj)
Definition: lwgeom.c:239
int SHPAPI_CALL DBFGetRecordCount(DBFHandle psDBF)
Definition: dbfopen.c:1205
#define SHPT_ARCZ
Definition: shapefil.h:313
void ShpLoaderDestroy(SHPLOADERSTATE *state)
#define GEOGRAPHY_DEFAULT
void SHPAPI_CALL SHPGetInfo(SHPHandle hSHP, int *pnEntities, int *pnShapeType, double *padfMinBound, double *padfMaxBound)
Definition: shpopen.c:797
int utf8(const char *fromcode, char *inputbuf, char **outputbuf)
#define POLICY_NULL_ABORT
char * escape_insert_string(char *str)
Escape input string suitable for INSERT.
int stringbuffer_aprintf(stringbuffer_t *s, const char *fmt,...)
Appends a formatted string to the current string buffer, using the format and argument list provided...
Definition: stringbuffer.c:247
SHPHandle SHPAPI_CALL SHPOpen(const char *pszShapeFile, const char *pszAccess)
Definition: shpopen.c:465
DBFFieldType SHPAPI_CALL DBFGetFieldInfo(DBFHandle psDBF, int iField, char *pszFieldName, int *pnWidth, int *pnDecimals)
Definition: dbfopen.c:1218
#define MAXFIELDNAMELEN
#define FLAGS_SET_Z(flags, value)
Definition: liblwgeom.h:112
int SHPAPI_CALL DBFGetFieldCount(DBFHandle psDBF)
Definition: dbfopen.c:1192
DBFHandle SHPAPI_CALL DBFOpen(const char *pszFilename, const char *pszAccess)
Definition: dbfopen.c:366
void set_loader_config_defaults(SHPLOADERCONFIG *config)
#define GEOMETRY_DEFAULT
LWGEOM * lwline_as_lwgeom(const LWLINE *obj)
Definition: lwgeom.c:249
#define FORCE_OUTPUT_2D
int ShpLoaderGetRecordCount(SHPLOADERSTATE *state)
int ptarray_append_point(POINTARRAY *pa, const POINT4D *pt, int allow_duplicates)
Append a point to the end of an existing POINTARRAY If allow_duplicate is LW_TRUE, then a duplicate point will not be added.
Definition: ptarray.c:141
void stringbuffer_clear(stringbuffer_t *s)
Reset the stringbuffer_t.
Definition: stringbuffer.c:84
#define LW_FALSE
Definition: liblwgeom.h:52
int FindPolygons(SHPObject *obj, Ring ***Out)
struct struct_point Point
#define LW_TRUE
Return types for functions with status returns.
Definition: liblwgeom.h:51
LWLINE * lwline_construct(int srid, GBOX *bbox, POINTARRAY *points)
Definition: lwline.c:29
char * escape_copy_string(char *str)
Escape input string suitable for COPY.
#define SRID_UNKNOWN
Unknown SRID value.
Definition: liblwgeom.h:154
struct struct_ring * next
SHPLOADERSTATE * ShpLoaderCreate(SHPLOADERCONFIG *config)
int nParts
Definition: shapefil.h:345
char * s
Definition: cu_in_wkt.c:24
#define SHPLOADEROK
double z
Definition: liblwgeom.h:308
#define WKB_EXTENDED
Definition: liblwgeom.h:1769
#define UTF8_NO_RESULT
#define WKT_PRECISION
#define SHPT_MULTIPOINTM
Definition: shapefil.h:319
#define FORCE_OUTPUT_DISABLE
#define ENCODING_DEFAULT
#define FORCE_OUTPUT_4D
#define SHPT_POINTZ
Definition: shapefil.h:312
#define MULTIPOLYGONTYPE
Definition: liblwgeom.h:65
SHPHandle hSHPHandle
Point * list
#define SHPT_POINTM
Definition: shapefil.h:316
void stringbuffer_append(stringbuffer_t *s, const char *a)
Append the specified string to the stringbuffer_t.
Definition: stringbuffer.c:128
#define WKT_EXTENDED
Definition: liblwgeom.h:1778
#define SHPT_POLYGONZ
Definition: shapefil.h:314
char message[SHPLOADERMSGLEN]
void ReleasePolygons(Ring **polys, int npolys)
#define SHPT_POINT
Definition: shapefil.h:308
void stringbuffer_destroy(stringbuffer_t *s)
Free the stringbuffer_t and all memory managed within it.
Definition: stringbuffer.c:72
#define POINTTYPE
LWTYPE numbers, used internally by PostGIS.
Definition: liblwgeom.h:60
LWPOLY * lwpoly_construct_empty(int srid, char hasz, char hasm)
Definition: lwpoly.c:66
void SHPAPI_CALL DBFClose(DBFHandle psDBF)
Definition: dbfopen.c:579
double * padfZ
Definition: shapefil.h:352
const char * stringbuffer_getstring(stringbuffer_t *s)
Returns a reference to the internal string being managed by the stringbuffer.
Definition: stringbuffer.c:143
#define MAXVALUELEN
LWGEOM * lwpoint_as_lwgeom(const LWPOINT *obj)
Definition: lwgeom.c:254
int ShpLoaderGetSQLCopyStatement(SHPLOADERSTATE *state, char **strheader)
int lwpoly_add_ring(LWPOLY *poly, POINTARRAY *pa)
Add a ring, allocating extra space if necessary.
Definition: lwpoly.c:154
int ShpLoaderGenerateSQLRowStatement(SHPLOADERSTATE *state, int item, char **strrecord)
int SHPAPI_CALL DBFIsAttributeNULL(DBFHandle psDBF, int iRecord, int iField)
Definition: dbfopen.c:1172
void strtolower(char *s)
void SHPAPI_CALL SHPClose(SHPHandle hSHP)
Definition: shpopen.c:760
LWPOINT * lwpoint_construct(int srid, GBOX *bbox, POINTARRAY *point)
Definition: lwpoint.c:96
#define SHPLOADERERR
int nSHPType
Definition: shapefil.h:341
double y
Definition: liblwgeom.h:308
unsigned int linked
#define MULTILINETYPE
Definition: liblwgeom.h:64
#define LWDEBUGF(level, msg,...)
Definition: lwgeom_log.h:55
#define SHPT_ARC
Definition: shapefil.h:309
double * padfM
Definition: shapefil.h:353
#define SHPLOADERMSGLEN
tuple y
Definition: pixval.py:54
#define COLLECTIONTYPE
Definition: liblwgeom.h:66
int SHPAPI_CALL DBFIsRecordDeleted(DBFHandle psDBF, int iShape)
Definition: dbfopen.c:1729
#define FLAGS_SET_M(flags, value)
Definition: liblwgeom.h:113
LWGEOM * lwcollection_as_lwgeom(const LWCOLLECTION *obj)
Definition: lwgeom.c:219