pythonxmlkmlsax

Python XML Sax Truncates String with no Special Characters


I downloaded some US census area file in KML format. You can download the file here. I am trying to grab the area name and the coordinate boundaries. For some reason, some of the coordinate fields are truncated and not read correctly. For example, the coordinates for "Bloomsburg-Berwick-Sunbury, PA" appears in the KML file as

<coordinates>-77.36418,40.846937,0.0 -77.357113,40.844484,0.0 -77.356628,40.807334,0.0 -77.354097,40.701667,0.0 -77.287941,40.693595,0.0 -77.150516,40.677074,0.0 -77.109453,40.691552,0.0 -77.093607,40.676121,0.0 -77.060451,40.679854,0.0 -77.035549,40.676918,0.0 -77.034409,40.659928,0.0 -77.008418,40.659912,0.0 -76.996995,40.635778,0.0 -76.965528,40.647149,0.0 -76.944828,40.650209,0.0 -76.939883,40.638142,0.0 -76.949148,40.628167,0.0 -76.918672,40.603466,0.0 -76.886411,40.617758,0.0 -76.864254,40.627585,0.0 -76.840104,40.625439,0.0 -76.810269,40.634526,0.0 -76.810044,40.640102,0.0 -76.804867,40.646839,0.0 -76.793851,40.640514,0.0 -76.745894,40.654464,0.0 -76.701624,40.658082,0.0 -76.700546,40.663114,0.0 -76.662137,40.674013,0.0 -76.562175,40.709007,0.0 -76.469523,40.743188,0.0 -76.380334,40.775445,0.0 -76.30717,40.801809,0.0 -76.2991,40.831191,0.0 -76.284611,40.883588,0.0 -76.207827,40.94974,0.0 -76.231194,41.050168,0.0 -76.228975,41.138466,0.0 -76.277639,41.131804,0.0 -76.317953,41.205453,0.0 -76.319957,41.211255,0.0 -76.310261,41.310198,0.0 -76.407934,41.308418,0.0 -76.447597,41.275629,0.0 -76.592607,41.157765,0.0 -76.640767,41.155718,0.0 -76.678776,41.154172,0.0 -76.732672,41.17204,0.0 -76.790807,41.175732,0.0 -76.828168,41.16578,0.0 -76.880963,41.158044,0.0 -76.884245,41.157099,0.0 -76.885228,41.155973,0.0 -76.888145,41.153807,0.0 -76.889338,41.151988,0.0 -76.889669,41.150791,0.0 -76.896114,41.13907,0.0 -76.960229,41.148801,0.0 -76.977939,41.087883,0.0 -77.058088,41.085575,0.0 -77.113839,41.069032,0.0 -77.144111,41.06884,0.0 -77.14416,41.044338,0.0 -77.204027,40.99271,0.0 -77.279236,40.90971,0.0 -77.36418,40.846937,0.0</coordinates>

But is truncated at character 297 out of 1664. This happens seemingly randomly for others as well. Size doesn't seem to be an issue.

['-77.36418,40.846937,0.0 -77.357113,40.844484,0.0 -77.356628,40.807334,0.0 -77.354097,40.701667,0.0 -77.287941,40.693595,0.0 -77.150516,40.677074,0.0 -77.109453,40.691552,0.0 -77.093607,40.676121,0.0 -77.060451,40.679854,0.0 -77.035549,40.676918,0.0 -77.034409,40.659928,0.0 -77.00841']

I tried on two different ec2 machines so I don't think it's a memory/hardware issue. Any idea what is going on?

from xml.sax.handler import ContentHandler
from xml.sax import parse
class KMLHandler(ContentHandler):
    def __init__(self):
        super().__init__()
        self.place_names = []
        self.current_name = None
        self.coordinates = []
        self.temp_coordinates = []
        self.start_placemark = False
        self.capture_place_name = False
        self.capture_cordinates = False
        self.mapping_dict = {}

    def startElement(self, name, attrs):
        if name == 'Placemark':
            self.first_placemark = True
            self.start_placemark = True
            self.temp_coordinates = []
            self.current_name = None
        else:
            pass
        if name == "SimpleData":
            if attrs['name'] == "NAME":
                self.capture_place_name = True
        if name == "coordinates":
            self.capture_cordinates = True

    def endElement(self, name):
        if name == "Placemark":
            self.start_placemark = False
            self.coordinates.append(self.temp_coordinates)
            self.mapping_dict[self.current_name] = self.temp_coordinates

    def characters(self, content):
        if content.strip() != "":
            if self.capture_place_name == True:
                self.place_names.append(content)
                self.current_name = content
                self.capture_place_name = False
            if self.capture_cordinates == True:
                str_vals = [x.split(',')[0:2] for x in content.split(' ')]
                self.temp_coordinates.append(content)
                self.capture_cordinates = False

fname='./cb_2020_us_csa_5m.kml'
# fname='./test_small2.kml'

handler = KMLHandler()
parse(fname, handler)

Solution

  • As indicated in the comments, each characters event returns a chunk, which may or may not be the entire tag contents. It's similar to reading from a network; you might not get everything at once.

    I reworked your code below, and it seems to report the right answer for Berwick. On my machine, the first chunk is 283 characters and the 2nd chunk is 1353 characters. 283 + 1353 = 1636, which matches the size of the data in the file.

    Instead of a set of Booleans, I think it's simpler to capture the tag name, and then test for that when you're processing characters. There's only one controlling value, and it's set & reset in one place.

    I didn't see a need for temp_coordinates. It wasn't clear to me whether you want coordinates to be a list or what, exactly, so I just grab the string.

    from xml.sax import parse
    class KMLHandler(ContentHandler):
        def __init__(self):
            super().__init__()
            self.place_names = []
            self.current_name = None
            self.coordinates = []
            self.start_placemark = False
            self.capture_place_name = False
            self.mapping_dict = {}
            self.capture = ''
    
        def startElement(self, name, attrs):
            self.capture = ''
            if name == 'Placemark':
                self.first_placemark = True
                self.start_placemark = True
                self.current_name = None
            else:
                pass
            if name == "SimpleData":
                if attrs['name'] == "NAME":
                    self.capture = name
            if name == "coordinates":
                self.capture = name
    
        def endElement(self, name):
            if name == "Placemark":
                self.start_placemark = False
                self.mapping_dict[self.current_name] = self.coordinates
                self.coordinates = []
    
        def characters(self, content):
            if content.strip() != "":
                if self.capture == 'SimpleData':
                    self.place_names.append(content)
                    self.current_name = content
                    self.capture_place_name = False
                if self.capture == "coordinates":
                    self.coordinates.append(content)
                    print( '%d coordinates for %s: {%s}' % (len(content),
                                                            self.current_name,
                                                            self.coordinates) )
    
    fname='./cb_2020_us_csa_5m.kml'
    # fname='./test_small2.kml'
    
    handler = KMLHandler()
    parse(fname, handler)