Page through large reads

export VIKE_API_KEY=...
python paging-large-reads.py             # real run: needs the key
python paging-large-reads.py --dry-run   # prints the first request of each walk, calls nothing

"""The four ways /v1 pages, each walked for a few pages: a cursor, a cursor in an envelope, an offset and a
moving time window.

Run:  VIKE_API_KEY=... python paging-large-reads.py    (--dry-run lists the requests and makes none)
A real walk drops MAX_PAGES and runs to the end.
"""
import os
import sys
from urllib.parse import urlencode

NAME = "paging-large-reads"
BASE = "https://data.vike.io"
DRY = "--dry-run" in sys.argv
MAX_PAGES = 3  # keeps the demo short; a real walk loops until the stop condition alone ends it
DAY_START = 1786924800000  # 2026-08-17T00:00:00Z in epoch ms
MINUTE_MS = 60_000


def fail(why):
    sys.exit(f"{NAME}: {why}")


def call(path, method="GET", **params):
    url = BASE + path + ("?" + urlencode(params) if params else "")
    if DRY:
        print(method, url)
        return None
    key = os.environ.get("VIKE_API_KEY") or fail("set VIKE_API_KEY")
    import requests

    r = requests.request(method, url, headers={"X-API-Key": key}, timeout=60)
    if r.status_code != 200:
        fail(f"{method} {path} answered HTTP {r.status_code}: {r.text[:200]}")
    return r.json()


def by_cursor():
    """HL liquidations: pass nextCursor back verbatim, with every other parameter unchanged, until it is null."""
    rows, cursor = 0, ""
    for _ in range(MAX_PAGES):
        params = dict(symbol="BTC", start="2026-08-17T00:00:00Z", end="2026-08-18T00:00:00Z", limit=500)
        if cursor:
            params["cursor"] = cursor
        page = call("/v1/hyperliquid/liquidations", **params)
        if page is None:
            return 0
        rows += len(page["liquidations"])
        cursor = page["nextCursor"]
        # A full last page still carries a cursor; the page after it is empty.
        if not cursor or not page["liquidations"]:
            break
    return rows


def by_envelope():
    """Polymarket settlements: pass next_cursor back as cursor while has_more is true."""
    rows, cursor = 0, ""
    for _ in range(MAX_PAGES):
        params = dict(slug_prefix="btc-updown-5m-", limit=1000)
        if cursor:
            params["cursor"] = cursor
        page = call("/v1/polymarket/settlements/bulk", **params)
        if page is None:
            return 0
        rows += len(page["settlements"])
        # has_more is exact: the last page says false and carries next_cursor null, with no empty page after it.
        if not page["has_more"]:
            break
        cursor = page["next_cursor"]
    return rows


def by_offset():
    """Polymarket archive index: offset += rows received, until offset reaches total."""
    rows, offset = 0, 0
    for _ in range(MAX_PAGES):
        page = call("/v1/polymarket/archive/datasets", stream="trades", limit=100, offset=offset)
        if page is None:
            return 0
        rows += len(page["datasets"])
        offset += len(page["datasets"])
        # total is the filtered count before limit and offset, the only field that says when to stop.
        if not page["datasets"] or offset >= page["total"]:
            break
    return rows


def by_window():
    """HL candles have no cursor: restart one bar after the last candle you hold."""
    rows, start = 0, DAY_START
    for _ in range(MAX_PAGES):
        page = call("/v1/hyperliquid/ohlcv", symbol="BTC", interval="1m", start=start,
                    end=DAY_START + 6 * 3_600_000, limit=100)
        if page is None:
            return 0
        candles = page["candles"]
        rows += len(candles)
        if len(candles) < 100:  # a short page is the last one
            break
        start = candles[-1]["ts"] + MINUTE_MS
    return rows


def main():
    counts = {"cursor": by_cursor(), "envelope": by_envelope(), "offset": by_offset(), "window": by_window()}
    if DRY:
        return
    for style, n in counts.items():
        if not n:
            fail(f"the {style} walk returned no rows")
        print(f"{style:8} paging: {n} rows over at most {MAX_PAGES} pages")


if __name__ == "__main__":
    try:
        main()
    except Exception as e:  # a shape the recipe does not expect fails by name, not by traceback
        fail(f"{type(e).__name__}: {e}")