Batch Convert JSON to CSV

import csv
import json
from pathlib import Path


def flatten_dict(d: dict, parent_key: str = "", sep: str = ".") -> dict:
    """Flatten nested dict: {'a': {'b': 1}} -> {'a.b': 1}"""
    items = []
    for k, v in d.items():
        new_key = f"{parent_key}{sep}{k}" if parent_key else k
        if isinstance(v, dict):
            items.extend(flatten_dict(v, new_key, sep).items())
        elif isinstance(v, list):
            items.append((new_key, json.dumps(v)))
        else:
            items.append((new_key, v))
    return dict(items)


# Merge all JSON files from a directory
all_records = []
for json_file in sorted(Path("./api_responses").glob("*.json")):
    data = json.loads(json_file.read_text())
    if isinstance(data, list):
        all_records.extend(data)
    elif isinstance(data, dict) and "results" in data:
        all_records.extend(data["results"])

# Flatten and collect all column names
flat_records = [flatten_dict(r) for r in all_records]
all_keys = []
seen = set()
for record in flat_records:
    for key in record:
        if key not in seen:
            all_keys.append(key)
            seen.add(key)

# Write CSV
with open("merged_output.csv", "w", newline="", encoding="utf-8") as f:
    writer = csv.DictWriter(f, fieldnames=all_keys, extrasaction="ignore")
    writer.writeheader()
    writer.writerows(flat_records)

print(f"Merged {len(flat_records)} records from {len(list(Path('./api_responses').glob('*.json')))} files")

const fs = require('fs');
const path = require('path');

function flattenObject(obj, prefix = '', sep = '.') {
  const result = {};
  for (const [key, val] of Object.entries(obj)) {
    const newKey = prefix ? `${prefix}${sep}${key}` : key;
    if (val && typeof val === 'object' && !Array.isArray(val)) {
      Object.assign(result, flattenObject(val, newKey, sep));
    } else {
      result[newKey] = Array.isArray(val) ? JSON.stringify(val) : val;
    }
  }
  return result;
}

// Read all JSON files
const dir = './api_responses';
const allRecords = [];
for (const file of fs.readdirSync(dir).filter(f => f.endsWith('.json')).sort()) {
  const data = JSON.parse(fs.readFileSync(path.join(dir, file), 'utf-8'));
  if (Array.isArray(data)) allRecords.push(...data);
  else if (data.results) allRecords.push(...data.results);
}

// Flatten and collect headers
const flat = allRecords.map(r => flattenObject(r));
const headers = [...new Set(flat.flatMap(Object.keys))];

// Write CSV
const escape = v => `"${String(v ?? '').replace(/"/g, '""')}"`;
const rows = [headers.map(escape).join(',')];
for (const record of flat) {
  rows.push(headers.map(h => escape(record[h] ?? '')).join(','));
}
fs.writeFileSync('merged_output.csv', rows.join('\n'));
console.log(`Merged ${flat.length} records → merged_output.csv`);