Heavy Data Formats: JSONL, Parquet & Protobuf

JSON is human-readable and universal, but it's also the least efficient format for bulk AI data. When you're storing 10 million evaluation records, a training dataset with embeddings, or streaming data between microservices, the right format makes a 10x difference in storage cost and processing speed.

JSONL: Line-Delimited JSON

JSONL (JSON Lines) stores one JSON object per line. Unlike regular JSON arrays, JSONL files can be appended to incrementally, streamed line-by-line without loading the full file, and processed in parallel. This makes JSONL the standard format for AI training data, evaluation logs, and audit trails.

import json
                      from pathlib import Path
                      from typing import Iterator
                      
                      # Writing JSONL — stream results as they're generated
                      def write_eval_results(results_iter, output_path: Path) -> int:
                          """Write evaluation results to JSONL, one per line. Returns record count."""
                          count = 0
                          with open(output_path, "w", encoding="utf-8") as f:
                              for result in results_iter:
                                  f.write(json.dumps(result, ensure_ascii=False) + "\n")
                                  count += 1
                          return count
                      
                      
                      # Reading JSONL — stream without loading everything into memory
                      def read_jsonl(path: Path) -> Iterator[dict]:
                          """Stream records from a JSONL file one at a time."""
                          with open(path, encoding="utf-8") as f:
                              for line_num, line in enumerate(f, start=1):
                                  line = line.strip()
                                  if not line:
                                      continue
                                  try:
                                      yield json.loads(line)
                                  except json.JSONDecodeError as e:
                                      print(f"Line {line_num}: invalid JSON — {e}")
                      
                      
                      # Practical: batch processing a large JSONL training dataset
                      def process_training_data(
                          jsonl_path: Path,
                          batch_size: int = 100,
                      ) -> Iterator[list[dict]]:
                          """Process a large JSONL file in batches."""
                          batch = []
                          for record in read_jsonl(jsonl_path):
                              batch.append(record)
                              if len(batch) >= batch_size:
                                  yield batch
                                  batch = []
                          if batch:
                              yield batch  # final partial batch
                      
                      
                      # Count lines efficiently without reading all data
                      def count_jsonl_records(path: Path) -> int:
                          count = 0
                          with open(path, "rb") as f:  # binary mode for speed
                              for _ in f:
                                  count += 1
                          return count
                      
                      
                      # Example: writing LLM evaluation log
                      eval_log = Path("eval_log.jsonl")
                      sample_results = [
                          {"id": "p001", "score": 0.92, "model": "gpt-4o", "latency_ms": 1240},
                          {"id": "p002", "score": 0.78, "model": "gpt-4o", "latency_ms": 890},
                      ]
                      records_written = write_eval_results(iter(sample_results), eval_log)
                      print(f"Wrote {records_written} records")

Parquet: Columnar Storage for AI Datasets

Parquet is a columnar binary format. Instead of storing each row sequentially (like CSV), it stores each column's values together. This means reads that access only a few columns (common in analytics) are dramatically faster — you only read the columns you need from disk.

import pandas as pd
                      import pyarrow as pa
                      import pyarrow.parquet as pq
                      from pathlib import Path
                      
                      # Writing Parquet from pandas
                      def save_eval_dataset(records: list[dict], output_path: Path) -> None:
                          """Save evaluation data to Parquet with optimal column types."""
                          df = pd.DataFrame(records)
                      
                          # Optimize storage types
                          df["score"] = df["score"].astype("float32")          # 4 bytes vs 8
                          df["tokens"] = df["tokens"].astype("int32")           # 4 bytes vs 8
                          df["model"] = df["model"].astype("category")          # encode repeated strings
                      
                          # Snappy compression: fast, reasonable ratio (default)
                          df.to_parquet(output_path, compression="snappy", index=False)
                          print(f"Saved {len(df)} records to {output_path} "
                                f"({output_path.stat().st_size / 1024:.1f} KB)")
                      
                      
                      # Reading Parquet — columnar efficiency
                      def load_scores_only(parquet_path: Path) -> pd.Series:
                          """Read only the score column — skips all other data on disk."""
                          return pd.read_parquet(parquet_path, columns=["score"])["score"]
                      
                      
                      # Filtering at read time (pushdown predicates)
                      def load_high_quality(parquet_path: Path, min_score: float = 0.8) -> pd.DataFrame:
                          """Load only records with score >= min_score."""
                          df = pd.read_parquet(parquet_path)
                          return df[df["score"] >= min_score]
                      
                      
                      # Partitioned Parquet for very large datasets
                      def save_partitioned_by_model(df: pd.DataFrame, output_dir: Path) -> None:
                          """Save dataset partitioned by model — queries on a specific model read only that partition."""
                          table = pa.Table.from_pandas(df)
                          pq.write_to_dataset(
                              table,
                              root_path=str(output_dir),
                              partition_cols=["model"],
                              compression="snappy",
                          )
                          # Creates: output_dir/model=gpt-4o/data.parquet
                          #          output_dir/model=gpt-4o-mini/data.parquet
                      
                      
                      # Read only one partition
                      def load_model_results(output_dir: Path, model: str) -> pd.DataFrame:
                          model_path = output_dir / f"model={model}"
                          return pd.read_parquet(model_path)

Protobuf: Typed Binary for Service Pipelines

Protocol Buffers (Protobuf) are Google's schema-first binary serialization format. They're smaller and faster than JSON, and unlike Parquet, they're designed for streaming between services — not analytics. Think of them as C#'s BinaryFormatter but cross-language, efficient, and with schema evolution.

// embedding_service.proto — define once, generate Python/C#/Go code
                      syntax = "proto3";
                      
                      package ai;
                      
                      message EmbeddingRequest {
                        string request_id = 1;
                        repeated string texts = 2;
                        string model = 3;
                        int32 dimensions = 4;
                      }
                      
                      message EmbeddingResponse {
                        string request_id = 1;
                        repeated EmbeddingVector embeddings = 2;
                        int32 input_tokens = 3;
                        float latency_ms = 4;
                      }
                      
                      message EmbeddingVector {
                        string text_id = 1;
                        repeated float values = 2;
                        int32 token_count = 3;
                      }

The .proto file is the schema — the single source of truth, much like a shared contract assembly. You don't hand-write the classes; you run the Protobuf compiler (protoc) to generate them, and it can emit Python, C#, Go, and more from the same file. That's what makes Protobuf a good fit for services written in different languages talking to each other.

# Generate Python classes from .proto file
                      pip install grpcio-tools
                      python -m grpc_tools.protoc -I. --python_out=. --grpc_python_out=. embedding_service.proto

That command produces an embedding_service_pb2 module containing typed classes for each message. From there it feels like using generated DTOs: construct an object, call SerializeToString() to get compact bytes for the wire, and FromString() to parse them back — with nested messages handled automatically. The size print shows the payoff over JSON.

# Using generated Protobuf classes
                      from embedding_service_pb2 import EmbeddingRequest, EmbeddingResponse, EmbeddingVector
                      
                      # Serialize to bytes
                      request = EmbeddingRequest(
                          request_id="req_abc123",
                          texts=["What is RAG?", "Explain embeddings"],
                          model="text-embedding-3-small",
                          dimensions=1536,
                      )
                      serialized: bytes = request.SerializeToString()
                      print(f"Size: {len(serialized)} bytes vs JSON: {len(str(request).encode())} bytes")
                      
                      # Deserialize
                      parsed = EmbeddingRequest.FromString(serialized)
                      print(parsed.request_id)   # "req_abc123"
                      print(list(parsed.texts))  # ["What is RAG?", "Explain embeddings"]
                      
                      # Nested messages
                      response = EmbeddingResponse(
                          request_id="req_abc123",
                          embeddings=[
                              EmbeddingVector(text_id="t0", values=[0.1, 0.2, 0.3], token_count=5),
                              EmbeddingVector(text_id="t1", values=[0.4, 0.5, 0.6], token_count=3),
                          ],
                          input_tokens=8,
                          latency_ms=45.2,
                      )

Format Decision Guide

"""
                      Quick decision rules for AI engineering:
                      
                      "I'm logging LLM evaluation results as they happen"
                        → JSONL (append-only, streamable, human-readable in pinch)
                      
                      "I'm storing a 1M-row evaluation dataset for analysis"
                        → Parquet (10-20x smaller than JSON, fast columnar reads)
                      
                      "I'm sending embeddings between a Python service and a Go service at 10k req/s"
                        → Protobuf (schema-enforced, ~5x smaller than JSON, typed)
                      
                      "I need to export results for a stakeholder to open in Excel"
                        → CSV (only valid use case for CSV in AI engineering)
                      
                      "I'm passing a config object or API payload"
                        → JSON (human-readable, universal, small size acceptable)
                      """