{ "name": "Warconomy free data sources roadmap", "asOf": "2026-06-05", "page": "https://warconomy.com/methodology/data-sources", "note": "An editorial audit of free public datasets/APIs. Policy: free + legally usable only; no committed API keys; no fragile live dependencies in the default build. Network-dependent sources are on-demand (committed snapshot) or detect-only.", "policy": [ "No paid APIs and no committed API keys or secrets.", "No runtime services or background workers; the default build makes no external network calls.", "Network-dependent ingests are on-demand and produce a committed static snapshot, kept out of default validation.", "Noisy/news-derived sources are for topic/report discovery only — never auto-published as facts.", "Sensitive casualty/displacement sources (UNHCR, IOM, UCDP, WHO) are citation-only — linked as estimates, never republished as Warconomy figures." ], "optionalKeyFramework": { "summary": "EIA and FRED expose useful data behind a free API key. They are classified as OPTIONAL-KEY FUTURE INTEGRATIONS: useful later, not blocking now, and never a default-build dependency.", "rules": [ "The key lives only in a local .env (or shell env); it is never committed.", "The ingest command reads the key from the environment and FAILS GRACEFULLY with a clear message if it is missing.", "Only the generated snapshot is committed — after review — never the key.", "A test asserts no API key/secret is committed anywhere in the repo.", "The default build and validation never require the key or a network call." ], "status": "not started — no EIA/FRED key handling exists yet; recorded as a future option only." }, "counts": { "total": 21, "ingestNow": 6, "watch": 6, "detectOnly": 2, "citationOnly": 5, "reject": 2 }, "decisions": { "ingestNow": [ "worldbank-api", "worldbank-pinksheet", "fao-fpi", "worldbank-demographics", "population-structure", "sipri-nato" ], "watch": [ "eia-api", "fred", "imf-portwatch", "un-desa-wpp", "ilostat", "uncomtrade" ], "detectOnly": [ "reliefweb", "gdelt" ], "citationOnly": [ "usgs-mcs", "unhcr", "iom-dtm", "who-gho", "ucdp" ], "reject": [ "acled", "oecd-iea" ] }, "candidates": [ { "id": "worldbank-api", "name": "World Bank Indicators API", "publisher": "World Bank", "url": "https://api.worldbank.org/v2/", "coverage": "Country macro indicators (GDP, inflation, military expenditure % GDP, trade)", "dataType": "JSON/XML time series", "access": "free-keyfree-api", "cadence": "Annual (mostly)", "machineReadable": true, "termsRisk": "Low — open data, permissive terms with attribution.", "usefulness": "High — macro context for conflict economies, complements the Pink Sheet prices already used.", "buildAndForget": true, "decision": "ingest-now", "notes": "Key-free, stable, official. Integrated as an on-demand snapshot powering /conflict-economies. No key, no default-build network." }, { "id": "worldbank-pinksheet", "name": "World Bank Commodity Markets (Pink Sheet)", "publisher": "World Bank", "url": "https://www.worldbank.org/en/research/commodity-markets", "coverage": "Monthly commodity price levels (energy, metals, food, fertilizer)", "dataType": "XLSX download", "access": "free-download", "cadence": "Monthly", "machineReadable": true, "termsRisk": "Low — open data with attribution.", "usefulness": "High — already powers the commodity history charts.", "buildAndForget": true, "decision": "ingest-now", "notes": "Already integrated (npm run commodities:ingest:worldbank); committed snapshot." }, { "id": "fao-fpi", "name": "FAO Food Price Index", "publisher": "FAO", "url": "https://www.fao.org/worldfoodsituation/foodpricesindex/en/", "coverage": "Global food price index + sub-indices", "dataType": "CSV download", "access": "free-download", "cadence": "Monthly", "machineReadable": true, "termsRisk": "Low — FAO open terms with attribution.", "usefulness": "High — already powers the food price charts.", "buildAndForget": true, "decision": "ingest-now", "notes": "Already integrated (npm run commodities:ingest); committed snapshot." }, { "id": "eia-api", "name": "EIA Open Data API", "publisher": "U.S. Energy Information Administration", "url": "https://www.eia.gov/opendata/", "coverage": "Oil/gas prices, production, stocks, chokepoint flows", "dataType": "JSON API", "access": "free-keyed-api", "cadence": "Daily–monthly", "machineReadable": true, "termsRisk": "Low — US government, public domain; but requires a free API key.", "usefulness": "High — authoritative energy data already cited via published EIA pages.", "buildAndForget": false, "decision": "watch", "notes": "Optional-key future integration (see the optional-key framework below). Never commit a key; the default build must not require one. EIA's published tables are already cited without the API." }, { "id": "fred", "name": "FRED (Federal Reserve Economic Data)", "publisher": "Federal Reserve Bank of St. Louis", "url": "https://fred.stlouisfed.org/", "coverage": "Macro and price series (incl. some commodity/energy)", "dataType": "JSON API / CSV", "access": "free-keyed-api", "cadence": "Varies", "machineReadable": true, "termsRisk": "Low — free, but API needs a key.", "usefulness": "Medium — overlaps World Bank/EIA for our needs.", "buildAndForget": false, "decision": "watch", "notes": "Optional-key future integration (see framework below). Keep the key uncommitted and out of the default build; some CSV endpoints work key-free on demand." }, { "id": "imf-portwatch", "name": "IMF PortWatch", "publisher": "International Monetary Fund", "url": "https://portwatch.imf.org/", "coverage": "Port and chokepoint trade disruption indicators", "dataType": "Portal + downloadable data", "access": "free-download", "cadence": "Frequent", "machineReadable": true, "termsRisk": "Low–medium — check redistribution terms; cite/link rather than republish.", "usefulness": "High — shipping-disruption context for chokepoint briefings.", "buildAndForget": false, "decision": "watch", "notes": "Already cited as a source. Linking/discovery is safe now; numeric ingest would need a reviewed snapshot." }, { "id": "usgs-mcs", "name": "USGS Mineral Commodity Summaries", "publisher": "U.S. Geological Survey", "url": "https://www.usgs.gov/centers/national-minerals-information-center/mineral-commodity-summaries", "coverage": "Production, reserves and trade for critical minerals (cobalt, copper, uranium, etc.)", "dataType": "PDF + downloadable data/CSV", "access": "free-download", "cadence": "Annual", "machineReadable": true, "termsRisk": "Low — US government, public domain.", "usefulness": "High — the authoritative figures behind the critical-minerals briefings.", "buildAndForget": false, "decision": "citation-only", "notes": "USGS pages block automated requests (HTTP 403) and data is annual PDF / ZIP-CSV per commodity — not a clean key-free API. Used citation-only: the /minerals page links USGS per-commodity pages and publishes no figures without a reviewed source packet. Do not scrape or hand-transcribe." }, { "id": "worldbank-demographics", "name": "World Bank demographic & labour indicators", "publisher": "World Bank", "url": "https://data.worldbank.org/", "coverage": "Population, ages 15–64, labour force, fertility, net migration, GDP, health spend", "dataType": "JSON API (key-free)", "access": "free-keyfree-api", "cadence": "Annual", "machineReadable": true, "termsRisk": "Low — open data with attribution.", "usefulness": "High — demographic context for the human-capital layer.", "buildAndForget": true, "decision": "ingest-now", "notes": "Integrated as an on-demand snapshot powering /human-capital (npm run worldbank:humancapital). Macro/demographic context only — NOT casualty data." }, { "id": "population-structure", "name": "Population age structure & dependency (UN WPP via World Bank)", "publisher": "World Bank / UN DESA", "url": "https://population.un.org/wpp/", "coverage": "Age bands (0–14, 15–64, 65+), working-age & dependency ratios for conflict economies", "dataType": "JSON API (key-free)", "access": "free-keyfree-api", "cadence": "Annual", "machineReadable": true, "termsRisk": "Low — UN WPP estimates republished by World Bank open data, with attribution.", "usefulness": "High — age-structure / dependency context powering /human-capital population pyramids.", "buildAndForget": true, "decision": "ingest-now", "notes": "Integrated via npm run population:ingest (committed snapshot). UN DESA's Data Portal API is reachable but heavy/less stable for build-and-forget, so the same UN-WPP estimates are taken from the key-free World Bank API. Demographic estimates only — NOT casualty data." }, { "id": "unhcr", "name": "UNHCR Refugee Data Finder", "publisher": "UNHCR (UN Refugee Agency)", "url": "https://www.unhcr.org/refugee-statistics/", "coverage": "Refugees, asylum-seekers, returnees by country/year (+ an API)", "dataType": "API / CSV", "access": "free-keyfree-api", "cadence": "Annual + updates", "machineReadable": true, "termsRisk": "Low — UN open data; sensitive subject, cite carefully and as estimates.", "usefulness": "High — the authoritative refugee figures for the human-capital briefings.", "buildAndForget": false, "decision": "citation-only", "notes": "Cite/link rather than republish. Displacement counts are estimates that change over time; do not present as Warconomy figures or as certainties." }, { "id": "iom-dtm", "name": "IOM Displacement Tracking Matrix", "publisher": "International Organization for Migration (IOM)", "url": "https://dtm.iom.int/", "coverage": "Internal displacement (IDP) tracking by country/crisis", "dataType": "Portal / downloads / API", "access": "free-keyfree-api", "cadence": "Frequent", "machineReadable": true, "termsRisk": "Low–medium — check redistribution; sensitive subject.", "usefulness": "High — IDP context that World Bank macro data cannot provide.", "buildAndForget": false, "decision": "citation-only", "notes": "Link for displacement context; estimates only, not republished as Warconomy figures." }, { "id": "un-desa-wpp", "name": "UN DESA World Population Prospects", "publisher": "UN DESA Population Division", "url": "https://population.un.org/wpp/", "coverage": "Population estimates and long-run projections by country", "dataType": "CSV / Excel / API", "access": "free-download", "cadence": "Periodic revisions", "machineReadable": true, "termsRisk": "Low — UN open data with attribution.", "usefulness": "Medium — long-run population projections beyond World Bank's horizon.", "buildAndForget": true, "decision": "watch", "notes": "Strong candidate for a future reviewed snapshot of population/age-structure; overlaps World Bank for current values." }, { "id": "who-gho", "name": "WHO Global Health Observatory", "publisher": "World Health Organization (WHO)", "url": "https://www.who.int/data/gho", "coverage": "Health indicators incl. disability, mental health, health systems", "dataType": "API / downloads", "access": "free-keyfree-api", "cadence": "Periodic", "machineReadable": true, "termsRisk": "Low — WHO open data; sensitive health topics, cite carefully.", "usefulness": "Medium — disability / mental-health context for veteran and PTSD briefings.", "buildAndForget": false, "decision": "citation-only", "notes": "Cite for health/disability context; do not derive clinical or prevalence claims." }, { "id": "ilostat", "name": "ILOSTAT (ILO labour statistics)", "publisher": "International Labour Organization (ILO)", "url": "https://ilostat.ilo.org/", "coverage": "Labour force, participation, employment, youth employment", "dataType": "API / bulk download", "access": "free-keyfree-api", "cadence": "Annual + updates", "machineReadable": true, "termsRisk": "Low — open data with attribution.", "usefulness": "Medium — labour detail beyond the World Bank headline series.", "buildAndForget": true, "decision": "watch", "notes": "Candidate for a future labour-force snapshot; World Bank covers the headline labour indicators for now." }, { "id": "ucdp", "name": "UCDP (Uppsala Conflict Data Program)", "publisher": "Uppsala University", "url": "https://ucdp.uu.se/", "coverage": "Organized-violence events and datasets", "dataType": "CSV / API", "access": "free-keyfree-api", "cadence": "Annual + candidate monthly", "machineReadable": true, "termsRisk": "Low — academic open data with citation.", "usefulness": "Medium — conflict-fatality context; Warconomy is economics-first, not a casualty tracker.", "buildAndForget": true, "decision": "citation-only", "notes": "Open academic data, usable. But fatality figures are sensitive and estimated — cite/link as ranges with caveats; never republish as Warconomy casualty counts or as certainties." }, { "id": "reliefweb", "name": "ReliefWeb API", "publisher": "UN OCHA", "url": "https://reliefweb.int/", "coverage": "Humanitarian situation reports, disasters, country pages", "dataType": "JSON API", "access": "free-keyfree-api", "cadence": "Continuous", "machineReadable": true, "termsRisk": "Low — UN open content; use for discovery/linking, not numeric claims.", "usefulness": "Medium — report discovery for crisis briefings (Sudan, DRC, Gaza).", "buildAndForget": false, "decision": "detect-only", "notes": "Use for report discovery/linking only; do NOT derive numeric claims. API endpoint shape changes — keep on-demand. (A tested endpoint returned 410, so treat as unstable.)" }, { "id": "gdelt", "name": "GDELT", "publisher": "GDELT Project", "url": "https://www.gdeltproject.org/", "coverage": "Global news event/tone index", "dataType": "API / bulk files", "access": "free-keyfree-api", "cadence": "Continuous (15-min)", "machineReadable": true, "termsRisk": "Medium — derived from news; noisy; attribution/quality vary.", "usefulness": "Low–medium — topic discovery only; not citable for facts.", "buildAndForget": false, "decision": "detect-only", "notes": "Event/topic discovery only. Too noisy to cite as a fact source; never auto-publish." }, { "id": "uncomtrade", "name": "UN Comtrade", "publisher": "United Nations", "url": "https://comtradeplus.un.org/", "coverage": "Bilateral trade flows by product", "dataType": "API / download", "access": "free-keyed-api", "cadence": "Annual/monthly", "machineReadable": true, "termsRisk": "Medium — free tier has rate/volume limits; premium for bulk.", "usefulness": "Medium — trade-flow context (minerals, semiconductors).", "buildAndForget": false, "decision": "watch", "notes": "Free tier is rate-limited; suitable for a small reviewed snapshot later, not default build." }, { "id": "acled", "name": "ACLED", "publisher": "ACLED", "url": "https://acleddata.com/", "coverage": "Armed conflict event data", "dataType": "API / download", "access": "free-keyed-api", "cadence": "Weekly", "machineReadable": true, "termsRisk": "Higher — registration + license terms restrict redistribution.", "usefulness": "Medium — high-quality events, but licensing is restrictive.", "buildAndForget": false, "decision": "reject", "notes": "Licensing/redistribution terms make it a poor fit for a static, openly-republished site. Prefer UCDP for open conflict context." }, { "id": "sipri-nato", "name": "SIPRI / NATO defense expenditure", "publisher": "SIPRI / NATO", "url": "https://www.sipri.org/databases/milex", "coverage": "Military expenditure (world, regional, national)", "dataType": "XLSX / PDF", "access": "free-download", "cadence": "Annual", "machineReadable": true, "termsRisk": "Low–medium — cite values; SIPRI terms restrict bulk redistribution.", "usefulness": "High — already cited for defense-spending pages.", "buildAndForget": true, "decision": "ingest-now", "notes": "Already used as cited sources (annual). Manual transcription of headline values, not bulk redistribution." }, { "id": "oecd-iea", "name": "OECD / IEA datasets", "publisher": "OECD / IEA", "url": "https://www.iea.org/data-and-statistics", "coverage": "Energy balances, prices, macro", "dataType": "API / download (mixed)", "access": "free-keyed-api", "cadence": "Varies", "machineReadable": true, "termsRisk": "Medium–high — much IEA data is paywalled or redistribution-restricted.", "usefulness": "Medium — overlaps free EIA/World Bank for our needs.", "buildAndForget": false, "decision": "reject", "notes": "Most useful IEA series are paywalled/restricted; free EIA and World Bank cover our needs without that risk." } ] }