DOM vs Streaming Parsers — XML Tutorial

Two ways to parse

	Tree (DOM, ElementTree, SimpleXML)	Streaming (SAX, StAX, iterparse, XMLReader)
Memory	The whole document, often 5 to 10 times the file size	Roughly constant
Access	Random: go anywhere, query with XPath	Forward-only, one pass
Code	Simple	More state to track
Use for	Configs, messages, documents up to tens of MB	Exports, feeds, logs, data dumps of any size

Streaming in three languages

The pattern: react to each finished product element, then free it so memory stays flat. The input here is small, but the same code handles gigabytes.

Stream through productsTry it yourself

import io
from lxml import etree

data = io.BytesIO(b"""<shop>
  <product sku="TEA-01" category="drinks"><name>Green tea</name><price>3.50</price><stock>40</stock></product>
  <product sku="COF-02" category="drinks"><name>Coffee beans</name><price>9.90</price><stock>0</stock></product>
  <product sku="MUG-07" category="kitchen"><name>Camel mug</name><price>12.00</price><stock>15</stock></product>
  <product sku="PAN-03" category="kitchen"><name>Frying pan</name><price>29.00</price><stock>4</stock></product>
</shop>""")
count, total = 0, 0.0
for _event, product in etree.iterparse(data, events=("end",), tag="product"):
    count += 1
    total += float(product.findtext("price")) * int(product.findtext("stock"))
    product.clear()                                   # free the element's children
    while product.getprevious() is not None:          # and earlier siblings
        del product.getparent()[0]
print(f"{count} products, stock value {total:.2f}")

Output

4 products, stock value 436.00

Stream through productsTry it yourself

<?php
$reader = XMLReader::XML('<shop>
  <product sku="TEA-01" category="drinks"><name>Green tea</name><price>3.50</price><stock>40</stock></product>
  <product sku="COF-02" category="drinks"><name>Coffee beans</name><price>9.90</price><stock>0</stock></product>
  <product sku="MUG-07" category="kitchen"><name>Camel mug</name><price>12.00</price><stock>15</stock></product>
  <product sku="PAN-03" category="kitchen"><name>Frying pan</name><price>29.00</price><stock>4</stock></product>
</shop>');
$count = 0;
$total = 0.0;
while ($reader->read()) {
    if ($reader->nodeType === XMLReader::ELEMENT && $reader->name === "product") {
        $product = simplexml_load_string($reader->readOuterXml());   // one small element at a time
        $count++;
        $total += (float) $product->price * (int) $product->stock;
    }
}
printf("%d products, stock value %.2f\n", $count, $total);

Output

4 products, stock value 436.00

Stream through products

import javax.xml.stream.*;
import java.io.StringReader;

public class Main {
    public static void main(String[] args) throws Exception {
        String xml = """
            <shop>
              <product sku="TEA-01" category="drinks"><name>Green tea</name><price>3.50</price><stock>40</stock></product>
              <product sku="COF-02" category="drinks"><name>Coffee beans</name><price>9.90</price><stock>0</stock></product>
              <product sku="MUG-07" category="kitchen"><name>Camel mug</name><price>12.00</price><stock>15</stock></product>
              <product sku="PAN-03" category="kitchen"><name>Frying pan</name><price>29.00</price><stock>4</stock></product>
            </shop>""";
        XMLStreamReader r = XMLInputFactory.newInstance().createXMLStreamReader(new StringReader(xml));
        int count = 0;
        double total = 0, price = 0;
        while (r.hasNext()) {
            if (r.next() != XMLStreamConstants.START_ELEMENT) continue;
            switch (r.getLocalName()) {
                case "product" -> count++;
                case "price" -> price = Double.parseDouble(r.getElementText());
                case "stock" -> total += price * Integer.parseInt(r.getElementText());
            }
        }
        System.out.printf("%d products, stock value %.2f%n", count, total);
    }
}

import javax.xml.stream.*;
import java.io.StringReader;

public class Main {
    public static void main(String[] args) throws Exception {
        String xml = """
            <shop>
              <product sku="TEA-01" category="drinks"><name>Green tea</name><price>3.50</price><stock>40</stock></product>
              <product sku="COF-02" category="drinks"><name>Coffee beans</name><price>9.90</price><stock>0</stock></product>
              <product sku="MUG-07" category="kitchen"><name>Camel mug</name><price>12.00</price><stock>15</stock></product>
              <product sku="PAN-03" category="kitchen"><name>Frying pan</name><price>29.00</price><stock>4</stock></product>
            </shop>""";
        XMLStreamReader r = XMLInputFactory.newInstance().createXMLStreamReader(new StringReader(xml));
        int count = 0;
        double total = 0, price = 0;
        while (r.hasNext()) {
            if (r.next() != XMLStreamConstants.START_ELEMENT) continue;
            switch (r.getLocalName()) {
                case "product" -> count++;
                case "price" -> price = Double.parseDouble(r.getElementText());
                case "stock" -> total += price * Integer.parseInt(r.getElementText());
            }
        }
        System.out.printf("%d products, stock value %.2f%n", count, total);
    }
}

Output

4 products, stock value 436.00

Run this on your computer — see “Get Started” for setup.

Common mistakes

Using iterparse but never clearing elements: the tree still grows to the full document.
Calling XMLReader::expand() or readOuterXml() on the root element: that loads everything again.
Building a DOM "just to check one value" in a high-traffic service; stream instead.

Test yourself

Why must you clear elements when using lxml's iterparse on a huge file?

Otherwise the parsed tree keeps growing and memory use approaches a full DOM

iterparse requires it to read the next element

To validate the document

To close the file

Show answer

Answer: Otherwise the parsed tree keeps growing and memory use approaches a full DOM

iterparse still builds the tree; clearing processed elements keeps memory constant.