Stream the data from the CGI script

test/largish.json.cgi

@@ -6,9 +6,6 @@ import sys
 
 random.seed(0)
 
-print("Content-Type: application/json")
-print()
-
 columns = [
     ('column0', lambda: 'XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX'),
     ('column1', lambda: random.choice([
@@ -20,14 +17,31 @@ columns = [
     ('column6', lambda: random.randint(0, 4)),
     ('column7', lambda: random.randint(0, 4))
 ]
-N_ROWS = 91746
 
-data = {}
+N_ROWS = 91746  # the output JSON size will be ~15 MB/10k rows
 
-for name, generator in columns:
-    column = {}
+
+class StreamDict(dict):
+    """
+    To serialize to JSON, we create an iterable object that inherits from a
+    known supported object type: dict.
+    """
+    def __init__(self, generator):
+        self.generator = generator
+
+    def items(self):
         for i in range(N_ROWS):
-        column[str(i)] = generator()
-    data[name] = column
+            yield i, self.generator()
+
+    def __len__(self):
+        return 1
 
+
+data = {}
+for name, generator in columns:
+    data[name] = StreamDict(generator)
+
+
+print("Content-Type: application/json")
+print()
 json.dump(data, sys.stdout)