magmalake arrow-mlake

arrow-mlake · API

arrow_mlake.arrow

Arrow types and Arrow array data, in the layout the C Data Interface wants.

ArrayData is one Arrow array: a type, a length, an optional validity bitmap, up to one offsets buffer, a values buffer, and children. Nesting is stored in an arena (ArrayArena) with children referenced by index, because Mojo does not allow a struct to contain a List of itself.

Buffer layouts follow the Arrow columnar spec exactly, so arrow_mlake.carrow can hand the buffers to another runtime without touching them:

type buffers
null —
boolean validity, values (bit-packed, LSB first)
fixed width validity, values
utf8 / binary validity, offsets (i32), data
large utf8 / large binary validity, offsets (i64), data
fixed-size binary, decimal128 validity, values
list / map validity, offsets (i32) + one child
large list validity, offsets (i64) + one child
struct validity + N children

Structs

ArrowType

struct ArrowType

One Arrow logical type. id is one of the AT_* constants.

Implements Copyable, Movable, Writable

Fields
id: Int
byte_width: Int: Element width for AT_FIXED_SIZE_BINARY and AT_DECIMAL128 (16).
precision: Int
scale: Int
unit: Int: Time unit for time/timestamp types, one of the TU_* constants.
tz: String: "UTC" for a UTC-adjusted timestamp, empty for a local one.
extension: String: An Arrow extension name such as arrow.uuid, or empty.

__init__

def __init__(out self, id: Int)
def __init__(out self)

is_nested

def is_nested(self) -> Bool

fixed_width

def fixed_width(self) -> Int

Bytes per element for a fixed-width type, or 0 if not fixed width.

format

def format(self) -> String

The Arrow C Data Interface format string for this type.

write_to

def write_to(self, mut writer: T)

ArrayData

struct ArrayData

One Arrow array. Children are indices into the owning ArrayArena.

Implements Copyable, Movable

Fields
type: ArrowType
name: String
nullable: Bool
field_id: Int32
length: Int
null_count: Int
validity: List[UInt8]
offsets: List[Int32]
large_offsets: List[Int64]
values: List[UInt8]
children: List[Int]

__init__

def __init__(out self, var type: ArrowType, var name: String)
def __init__(out self)

is_valid

def is_valid(self, i: Int) -> Bool

ArrayArena

struct ArrayArena

Owns every ArrayData of a batch; children are referenced by index.

Implements Copyable, Defaultable, Movable

Fields
nodes: List[ArrayData]

__init__

def __init__(out self)

add

def add(mut self, var node: ArrayData) -> Int

graft

def graft(mut self, mut other: Self, root: Int) -> Int

Move every node of other in here, keeping its order; return where root ended up.

This is what lets an array be built somewhere else and still land in a caller-chosen place in this arena. Children are indices, so a subtree built on its own — starting at index 0 — needs every one of them shifted by however many nodes are already here. Because the nodes keep their relative order, the result is byte for byte the arena that building the same subtree straight into this one would have produced.

That equivalence is the whole point: it is what makes it safe to build several top-level fields at once and concatenate them in field order afterwards, whatever order they actually finished in.

other is left holding that many empty arrays — an ArrayData owns its buffers, so this moves them out one slot at a time rather than copying.

__len__

def __len__(self) -> Int

Functions

unit_suffix

def unit_suffix(unit: Int) -> String

unit_name

def unit_name(unit: Int) -> String

at

def at(id: Int) -> ArrowType

at_fixed

def at_fixed(width: Int) -> ArrowType

at_decimal

def at_decimal(precision: Int, scale: Int) -> ArrowType

at_timestamp

def at_timestamp(unit: Int, var tz: String) -> ArrowType

at_time

def at_time(unit: Int) -> ArrowType

bitmap_bytes

def bitmap_bytes(n: Int) -> Int

bit_get

def bit_get(bitmap: Span[UInt8], i: Int) -> Bool

bit_fill_valid

def bit_fill_valid(mut bitmap: List[UInt8], count: Int)

Materialise a bitmap in which the first count entries are all valid.

bit_set

def bit_set(mut bitmap: List[UInt8], i: Int, value: Bool)

load_i32

def load_i32(buf: Span[UInt8], i: Int) -> Int32

load_i64

def load_i64(buf: Span[UInt8], i: Int) -> Int64

load_u64

def load_u64(buf: Span[UInt8], i: Int) -> UInt64

load_f32

def load_f32(buf: Span[UInt8], i: Int) -> Float32

load_f64

def load_f64(buf: Span[UInt8], i: Int) -> Float64

store_u32

def store_u32(mut buf: List[UInt8], v: UInt32)

store_u64

def store_u64(mut buf: List[UInt8], v: UInt64)

Aliases

comptime AT_NULL = 0
comptime AT_BOOL = 1
comptime AT_INT8 = 2
comptime AT_INT16 = 3
comptime AT_INT32 = 4
comptime AT_INT64 = 5
comptime AT_UINT8 = 6
comptime AT_UINT16 = 7
comptime AT_UINT32 = 8
comptime AT_UINT64 = 9
comptime AT_FLOAT16 = 10
comptime AT_FLOAT32 = 11
comptime AT_FLOAT64 = 12
comptime AT_UTF8 = 13
comptime AT_LARGE_UTF8 = 14
comptime AT_BINARY = 15
comptime AT_LARGE_BINARY = 16
comptime AT_FIXED_SIZE_BINARY = 17
comptime AT_DECIMAL128 = 18
comptime AT_DATE32 = 19
comptime AT_TIME32 = 20
comptime AT_TIME64 = 21
comptime AT_TIMESTAMP = 22
comptime AT_LIST = 23
comptime AT_LARGE_LIST = 24
comptime AT_STRUCT = 25
comptime AT_MAP = 26
comptime TU_SECOND = 0
comptime TU_MILLI = 1
comptime TU_MICRO = 2
comptime TU_NANO = 3