magmalake iceberg

iceberg · API

iceberg.metadata

TableMetadata — the whole of an Iceberg *.metadata.json, v1 through v3.

Every field in the spec's table-metadata table is parsed and re-serialized, including the v1-only singular schema / partition-spec forms, the v2/v3 plural schemas / partition-specs, refs, statistics, partition-statistics, the v3 next-row-id and encryption-keys, and the v3 snapshot fields first-row-id / added-rows / key-id.

Reading is deliberately forgiving in the directions the spec asks for: unrecognised top-level keys are kept verbatim in extra and written back out, an unknown format-version above 3 is rejected (the spec requires it), and a v4 metadata file's optional location is tolerated.

Snapshot selection lives here too — current_snapshot, snapshot_by_id, snapshot_for_ref, snapshot_as_of — along with filesystem-table discovery (version-hint.text plus v<N>.metadata.json).

Structs

Snapshot

struct Snapshot

One table snapshot.

Implements Copyable, Movable, Writable

Fields
snapshot_id: Int64
parent_snapshot_id: Int64
has_parent: Bool
sequence_number: Int64
timestamp_ms: Int64
manifest_list: String
manifests: List[String]: V1 only: an inline manifest list. Empty when manifest-list is set.
summary: Dict[String, String]
schema_id: Int
has_schema_id: Bool
first_row_id: Int64
has_first_row_id: Bool
added_rows: Int64
has_added_rows: Bool
key_id: String
has_key_id: Bool

operation

def operation(self) -> String

The required operation summary key; "append" when absent.

summary_int

def summary_int(self, key: String, dflt: Int64) -> Int64

write_to

def write_to(self, mut writer: T)

from_json

def from_json(doc: Json, i: Int, format_version: Int) -> Self

to_json

def to_json(self) -> String

SnapshotRef

struct SnapshotRef

A named branch or tag plus its retention policy.

Implements Copyable, Movable

Fields
name: String
snapshot_id: Int64
type: String
min_snapshots_to_keep: Int64
has_min_snapshots_to_keep: Bool
max_snapshot_age_ms: Int64
has_max_snapshot_age_ms: Bool
max_ref_age_ms: Int64
has_max_ref_age_ms: Bool

is_branch

def is_branch(self) -> Bool

from_json

def from_json(doc: Json, i: Int, var name: String) -> Self

to_json

def to_json(self) -> String

SnapshotLogEntry

struct SnapshotLogEntry

Implements Copyable, Movable

Fields
timestamp_ms: Int64
snapshot_id: Int64

MetadataLogEntry

struct MetadataLogEntry

Implements Copyable, Movable

Fields
timestamp_ms: Int64
metadata_file: String

BlobMetadata

struct BlobMetadata

Implements Copyable, Movable

Fields
type: String
snapshot_id: Int64
sequence_number: Int64
fields: List[Int]
properties: Dict[String, String]

StatisticsFile

struct StatisticsFile

A Puffin statistics file attached to one snapshot.

Implements Copyable, Movable

Fields
snapshot_id: Int64
statistics_path: String
file_size_in_bytes: Int64
file_footer_size_in_bytes: Int64
key_metadata: String
has_key_metadata: Bool
blob_metadata: List[BlobMetadata]

from_json

def from_json(doc: Json, i: Int) -> Self

to_json

def to_json(self) -> String

PartitionStatisticsFile

struct PartitionStatisticsFile

Implements Copyable, Movable

Fields
snapshot_id: Int64
statistics_path: String
file_size_in_bytes: Int64

to_json

def to_json(self) -> String

EncryptionKey

struct EncryptionKey

A v3 table-encryption key entry.

The spec's field table names the payload encrypted-key-metadata while its own prose calls it key-metadata; both spellings are accepted on read and the spec's table spelling is written back.

Implements Copyable, Movable

Fields
key_id: String
encrypted_key_metadata: String
encrypted_by_id: String
has_encrypted_by_id: Bool
properties: Dict[String, String]

from_json

def from_json(doc: Json, i: Int) -> Self

to_json

def to_json(self) -> String

TableMetadata

struct TableMetadata

A parsed *.metadata.json.

Implements Copyable, Movable

Fields
format_version: Int
table_uuid: String
location: String
has_location: Bool: V4 makes location optional; v1-v3 always carry it.
last_sequence_number: Int64
last_updated_ms: Int64
last_column_id: Int
schemas: List[Schema]
current_schema_id: Int
partition_specs: List[PartitionSpec]
default_spec_id: Int
last_partition_id: Int
properties: Dict[String, String]
current_snapshot_id: Int64
has_current_snapshot: Bool
snapshots: List[Snapshot]
snapshot_log: List[SnapshotLogEntry]
metadata_log: List[MetadataLogEntry]
sort_orders: List[SortOrder]
default_sort_order_id: Int
refs: List[SnapshotRef]
statistics: List[StatisticsFile]
partition_statistics: List[PartitionStatisticsFile]
next_row_id: Int64
has_next_row_id: Bool
encryption_keys: List[EncryptionKey]
extra_keys: List[String]: Top-level keys this build does not know, preserved for round-tripping.
extra_values: List[String]
metadata_file_location: String: Where this file was read from, when it was read from a file.

__init__

def __init__(out self)

schema

def schema(self) -> Schema

The table's current schema.

schema_by_id

def schema_by_id(self, id: Int) -> Schema

has_schema

def has_schema(self, id: Int) -> Bool

spec

def spec(self) -> PartitionSpec

spec_by_id

def spec_by_id(self, id: Int) -> PartitionSpec

has_spec

def has_spec(self, id: Int) -> Bool

sort_order_by_id

def sort_order_by_id(self, id: Int) -> SortOrder

snapshot_index

def snapshot_index(self, id: Int64) -> Int

snapshot_by_id

def snapshot_by_id(self, id: Int64) -> Snapshot

current_snapshot

def current_snapshot(self) -> Snapshot

for_reading

def for_reading(self) -> Self

A copy for reading the current snapshot, without the history.

It keeps the schemas, specs, sort orders and properties, the current snapshot and the refs that point at it, and drops the other snapshots, the snapshot and metadata logs and the statistics files. A table written by many commits carries all of them, so this is much smaller, and it's what to send to the workers reading one plan's files: they need the schema and the snapshot, not the table's past.

ref_index

def ref_index(self, name: String) -> Int

snapshot_for_ref

def snapshot_for_ref(self, name: String) -> Snapshot

Resolve a branch or tag. main falls back to current-snapshot-id, which the spec says is always the main branch's snapshot even when the refs map is absent.

snapshot_as_of

def snapshot_as_of(self, timestamp_ms: Int64) -> Snapshot

The snapshot that was current at timestamp_ms.

Resolved from snapshot-log — the record of which snapshot was current when — falling back to the snapshot list when the log is empty.

schema_for_snapshot

def schema_for_snapshot(self, s: Snapshot) -> Schema

The schema in effect for a snapshot: its own schema-id when it records one, else the table's current schema.

property_

def property_(self, key: String, dflt: String) -> String

parse

def parse(text: String) -> Self

from_json

def from_json(doc: Json, i: Int) -> Self

to_json

def to_json(self) -> String

Aliases

comptime MAIN_BRANCH = String("main")
comptime SUPPORTED_FORMAT_VERSION = Int(3)
comptime INITIAL_SPEC_ID = Int(0)
comptime INITIAL_SORT_ORDER_ID = Int(0)
comptime REF_BRANCH = String("branch")
comptime REF_TAG = String("tag")