Load a month of Hyperliquid fills into Polars or Postgres within the daily quota

import os
from pathlib import Path

import polars as pl
import requests

API = "https://data.vike.io/v1/hyperliquid"
H = {"X-API-Key": os.environ["VIKE_API_KEY"]}
MONTH = "2026-09"

index = requests.get(f"{API}/archive/datasets", headers=H, timeout=60).json()
days = sorted(d["date"] for d in index["dates"] if d["date"].startswith(MONTH) and "fills" in d["streams"])
need = sum(d["streams"]["fills"]["bytes"] for d in index["dates"] if d["date"] in days)
print(len(days), "days,", round(need / 1e9, 2), "GB of links")  # must fit 20 GB per rolling 24 hours

for day in days:
    out = Path(f"btc-{day}.parquet")
    if out.exists():
        continue  # done on an earlier run: no new link is asked for
    ask = requests.get(f"{API}/archive/url", params={"dataset": "fills", "date": day}, headers=H, timeout=60)
    ask.raise_for_status()  # a 429 here says how much was issued in the last 24 hours
    raw = Path(f"fills-{day}.parquet")
    with requests.get(ask.json()["url"], stream=True, timeout=(10, 300)) as r:  # the link needs no key
        r.raise_for_status()
        with raw.open("wb") as f:
            for chunk in r.iter_content(1 << 20):
                f.write(chunk)
    pl.scan_parquet(raw).filter(pl.col("asset") == "BTC").sink_parquet(out)  # keep only what you need
    raw.unlink()  # free the disk before the next day

month = pl.read_parquet(f"btc-{MONTH}-*.parquet")

{
  "url": "https://data.vike.io/dl/<exp>.<kid>.<sig>/venue=hyperliquid/date=2026-10-23/fills.parquet",
  "expires_at": "2026-10-24T17:22:08Z",
  "expires_in": 300,
  "dataset": "fills",
  "date": "2026-10-23",
  "bytes": 357514873,
  "rows": 11936174
}

pl.read_parquet("btc-2026-09-01.parquet").write_csv("btc-2026-09-01.csv", datetime_format="%Y-%m-%dT%H:%M:%S%.3f%:z")

CREATE TABLE hl_fills (
  fill_ts timestamptz, wallet text, asset text, side text,
  size_token double precision, price double precision, size_usd double precision,
  fee double precision, fee_token text, is_maker boolean,
  order_id numeric(20,0), trade_id numeric(20,0), crossed boolean,
  closed_pnl double precision, dex_name text, start_position double precision
);
\copy hl_fills FROM 'btc-2026-09-01.csv' WITH (FORMAT csv, HEADER true)