Download one archive day

export VIKE_API_KEY=...
python pm-download-archive-day.py             # real run: needs the key, writes polymarket-trades-<date>.parquet
python pm-download-archive-day.py --dry-run   # prints the requests it would make, downloads nothing

"""Download the newest day of the Polymarket taker tape from the bulk archive: one whole file, one request, size checked.

Run:  VIKE_API_KEY=... python pm-download-archive-day.py    (--dry-run lists the requests and makes none)
"""
import os
import sys
from urllib.parse import urlencode

NAME = "pm-download-archive-day"
BASE = "https://data.vike.io"
DRY = "--dry-run" in sys.argv


def fail(why):
    sys.exit(f"{NAME}: {why}")


def call(path, method="GET", **params):
    url = BASE + path + ("?" + urlencode(params) if params else "")
    if DRY:
        print(method, url)
        return None
    key = os.environ.get("VIKE_API_KEY") or fail("set VIKE_API_KEY")
    import requests

    r = requests.request(method, url, headers={"X-API-Key": key}, timeout=60)
    if r.status_code != 200:
        fail(f"{method} {path} answered HTTP {r.status_code}: {r.text[:200]}")
    return r.json()


def download(url, path):
    """The whole file in one request, streamed to disk. Never point a Parquet reader at the URL instead."""
    if DRY:
        print("GET", url)
        return
    import requests
    from urllib.parse import urlparse

    # The key must only ever go to our own host, whatever URL the API handed back.
    if urlparse(url).hostname != "data.vike.io":
        fail(f"refusing to send the API key to {url}")
    with requests.get(url, headers={"X-API-Key": os.environ["VIKE_API_KEY"]}, stream=True, timeout=300) as r:
        if r.status_code != 200:
            fail(f"GET {url} answered HTTP {r.status_code}")
        with open(path, "wb") as f:
            for chunk in r.iter_content(chunk_size=1 << 20):
                f.write(chunk)


def main():
    # Flat rows (every market that day) come first and newest first; family rows follow. Take the newest flat date.
    index = call("/v1/polymarket/archive/datasets", stream="trades", limit=1000)
    day = "2026-09-02"  # --dry-run has no response to read
    if not DRY:
        flat = [d["date"] for d in index["datasets"] if d["layout"] == "flat"]
        if not flat:
            fail("the archive index lists no flat trades file")
        day = max(flat)

    # The bytes and rows of that one file, so the download can be checked against them.
    meta = call("/v1/polymarket/archive/url", date=day, stream="trades")
    url = f"{BASE}/archive/venue=polymarket/date={day}/trades.parquet"  # --dry-run: the shape /archive/url returns
    if not DRY:
        url = meta["url"]
    path = f"polymarket-trades-{day}.parquet"
    download(url, path)
    if DRY:
        return
    got = os.path.getsize(path)
    if got != meta["bytes"]:
        fail(f"{path} is {got} bytes, the archive says {meta['bytes']} (truncated download?)")
    print(f"{path}: {got:,} bytes, {meta['rows']:,} rows, {meta['layout']} layout")


if __name__ == "__main__":
    try:
        main()
    except Exception as e:  # a shape the recipe does not expect fails by name, not by traceback
        fail(f"{type(e).__name__}: {e}")

import pyarrow.parquet as pq

table = pq.read_table("polymarket-trades-2026-09-02.parquet", columns=["ts", "token_id", "price", "size", "side"])