Formats

from astropy.table import Table

t = Table.read("https://asassn-v3.ifa.hawaii.edu/v3/stars/11010359/lc?format=ecsv", format="ascii.ecsv")
print(t["mag"].dtype, t["mag"].unit, t.meta["release_id"])

{"header": {"format": "SP3-JSON", "version": 1, "kind": "star_lc", "release_id": "…", "serve_id": "…",
            "object": {…}, "selection": {…}, "curves": […], "n": 3,
            "columns": [{"name": "bjd", "type": "f8", "unit": "d", "desc": "BJD_TDB at mid-exposure"}, …]},
 "columns": {"bjd": [2456024.1328295474, …], "mag": [14.036707, …], "camera": ["bA", …],
             "field": ["F0845+18_0428", …], …}}

"""Minimal SP3C v1 reader (numpy only). Sky Patrol V3 docs: /docs/formats.html"""
import json
import struct

import numpy as np

TYPES = {"f8": "<f8", "f4": "<f4", "i4": "<i4", "u4": "<u4", "i2": "<i2", "u2": "<u2", "u1": "u1"}


def read_sp3c(buf):
    """Return (header, columns) from one SP3C payload; str columns become lists of str."""
    buf = memoryview(buf).cast("B")
    if bytes(buf[:4]) != b"SP3C":
        raise ValueError("not an SP3C payload")
    (hlen,) = struct.unpack_from("<I", buf, 4)
    header = json.loads(bytes(buf[8:8 + hlen]))
    if int(str(header["version"]).split(".")[0]) != 1:
        raise ValueError("unsupported SP3C version %r" % header["version"])
    base, n, cols = 8 + hlen, header["n"], {}
    for c in header["columns"]:
        if c["offset"] % 8:
            raise ValueError("misaligned column %r" % c["name"])
        at = base + c["offset"]
        if c["type"] == "str":  # u4 offsets[n+1], then UTF-8 bytes
            offs = np.frombuffer(buf, "<u4", n + 1, at).tolist()
            blob = bytes(buf[at + 4 * (n + 1):at + c["nbytes"]])
            cols[c["name"]] = [blob[offs[i]:offs[i + 1]].decode("utf-8") for i in range(n)]
            continue
        if c["type"] not in TYPES:
            raise ValueError("unknown column type %r" % c["type"])
        dt = np.dtype(TYPES[c["type"]])
        if c.get("shuffled") and dt.itemsize > 1:  # itemsize planes of n bytes each
            planes = np.frombuffer(buf, np.uint8, n * dt.itemsize, at).reshape(dt.itemsize, n)
            cols[c["name"]] = np.ascontiguousarray(planes.T).view(dt).reshape(n)
        else:
            cols[c["name"]] = np.frombuffer(buf, dt, n, at)
    return header, cols


def expand(header, cols, name):
    """Strings for a dictionary-coded column such as 'camera' or 'band'."""
    spec = next(c for c in header["columns"] if c["name"] == name)
    return np.asarray(header["dictionaries"][spec["dict"]])[cols[name]]

// Minimal SP3C v1 reader for browsers and Node 18+. Sky Patrol V3 docs: /docs/formats.html
const TYPES = {f8: Float64Array, f4: Float32Array, i4: Int32Array, u4: Uint32Array,
               i2: Int16Array, u2: Uint16Array, u1: Uint8Array};

export function readSP3C(buf) {           // buf: ArrayBuffer, e.g. await (await fetch(url)).arrayBuffer()
  const dv = new DataView(buf);
  if (dv.getUint32(0, true) !== 0x43335053) throw new Error('not an SP3C payload');   // "SP3C"
  const hlen = dv.getUint32(4, true), base = 8 + hlen;
  const header = JSON.parse(new TextDecoder().decode(new Uint8Array(buf, 8, hlen)));
  if (parseInt(header.version, 10) !== 1) throw new Error('unsupported SP3C version ' + header.version);
  const n = header.n, cols = {};
  for (const c of header.columns) {
    if (c.offset % 8) throw new Error('misaligned column ' + c.name);
    const at = base + c.offset;
    if (c.type === 'str') {                // u4 offsets[n+1], then UTF-8 bytes
      const off = new Uint32Array(buf, at, n + 1), bytes = new Uint8Array(buf, at + 4 * (n + 1)), td = new TextDecoder();
      cols[c.name] = Array.from({length: n}, (_, i) => td.decode(bytes.subarray(off[i], off[i + 1])));
      continue;
    }
    const T = TYPES[c.type];
    if (!T) throw new Error('unknown column type ' + c.type);
    if (!c.shuffled || T.BYTES_PER_ELEMENT === 1) { cols[c.name] = new T(buf, at, n); continue; }  // zero-copy
    const w = T.BYTES_PER_ELEMENT, src = new Uint8Array(buf, at, n * w), out = new Uint8Array(n * w);
    for (let b = 0; b < w; b++) for (let i = 0; i < n; i++) out[i * w + b] = src[b * n + i];
    cols[c.name] = new T(out.buffer);
  }
  return {header, cols};
}

export function expand(header, cols, name) {   // strings of a dictionary column ('camera', 'band', ...)
  const dict = header.dictionaries[header.columns.find(c => c.name === name).dict];
  return Array.from(cols[name], k => dict[k]);
}

"""Minimal SP3F v1 stream reader. Sky Patrol V3 docs: /docs/formats.html"""
import json
import struct

from sp3c_reader import read_sp3c  # the SP3C reader on the same page


def _exact(stream, n):
    data = b""
    while len(data) < n:
        chunk = stream.read(n - len(data))
        if not chunk:
            raise EOFError("SP3F stream ended early: no trailer, treat the batch as failed")
        data += chunk
    return data


def read_sp3f(stream):
    """Yield ('item', header, cols), ('error', dict) and finally ('trailer', dict)."""
    head = _exact(stream, 8)
    if head[:4] != b"SP3F" or head[4] != 1:
        raise ValueError("not an SP3F v1 stream")
    while True:
        frame = _exact(stream, 8)
        kind, (length,) = frame[0], struct.unpack_from("<I", frame, 4)
        payload = _exact(stream, length + (-length) % 8)[:length]
        if kind == 1:
            header, cols = read_sp3c(payload)
            yield "item", header, cols
        elif kind == 2:
            yield "error", json.loads(payload)
        elif kind == 3:
            yield "trailer", json.loads(payload)
            return
        else:
            raise ValueError("unknown SP3F frame kind %d" % kind)