export type CursorPageResponse = { items: TItem[]; limit?: number; offset?: number; total?: number; cursor?: string; nextCursor?: string; next_cursor?: string; }; export type CursorFetchParams = Record< string, string | number | boolean | null | undefined >; export type CursorFetchRequest = { baseUrl: string; path: string; pageSize?: number; params?: CursorFetchParams; headers?: Record; credentials?: RequestCredentials; maxPages?: number; }; function normalizeBaseUrl(baseUrl: string) { const value = baseUrl.endsWith("/") ? baseUrl : `${baseUrl}/`; return new URL(value, globalThis.location?.origin ?? "http://localhost"); } function buildCursorFetchUrl( request: Required> & Pick, pageSize: number, cursor: string | undefined, ) { const path = request.path.replace(/^\/+/, ""); const url = new URL(path, normalizeBaseUrl(request.baseUrl)); for (const [key, value] of Object.entries(request.params ?? {})) { if (value !== undefined && value !== null && value !== "") { url.searchParams.set(key, String(value)); } } url.searchParams.set("limit", String(pageSize)); if (cursor) { url.searchParams.set("cursor", cursor); } else { url.searchParams.delete("cursor"); } return url; } function readNextCursor(page: CursorPageResponse) { return page.nextCursor || page.next_cursor || undefined; } export async function fetchAllCursorPagesMainThread({ pageSize = 100, credentials = "same-origin", maxPages = 1000, ...request }: CursorFetchRequest): Promise> { const items: TItem[] = []; let cursor: string | undefined; for (let pageIndex = 0; pageIndex < maxPages; pageIndex += 1) { const url = buildCursorFetchUrl(request, pageSize, cursor); const response = await fetch(url, { headers: request.headers, credentials, }); if (!response.ok) { throw new Error( `Cursor page request failed with status ${response.status}`, ); } const page = (await response.json()) as CursorPageResponse; items.push(...page.items); const nextCursor = readNextCursor(page); if (!nextCursor) { return { ...page, items, limit: pageSize, offset: 0, total: items.length, cursor, nextCursor: undefined, next_cursor: undefined, }; } if (nextCursor === cursor) { throw new Error("Cursor page request returned the same next cursor"); } cursor = nextCursor; } throw new Error(`Cursor page request exceeded ${maxPages} pages`); }