VectorFin Apache Iceberg Financial Data

pip install "pyiceberg[gcs]" pandas numpy

from pyiceberg.catalog import load_catalog

catalog = load_catalog(
    "vectorfin",
    **{
        "type": "rest",
        "uri": "https://catalog.vectorfinancials.com",
        "credential": "client_id:client_secret",  # from your VectorFin dashboard
        "warehouse": "vectorfin_warehouse",
    }
)

# List available namespaces and tables
print(catalog.list_namespaces())
# [('embeddings',), ('signals',)]

print(catalog.list_tables("signals"))
# [('signals', 'whystock_score'), ('signals', 'regime'), ...]

import pandas as pd

# Open the whystock_score signals table
table = catalog.load_table("signals.whystock_score")

# Scan with filters (pushdown predicates)
from pyiceberg.expressions import And, GreaterThanOrEqual, EqualTo

df = table.scan(
    row_filter=And(
        EqualTo("ticker", "AAPL"),
        GreaterThanOrEqual("date", "2024-01-01"),
    ),
    selected_fields=("ticker", "date", "score", "components"),
).to_pandas()

print(df.head(10))

from pyiceberg.expressions import And, LessThanOrEqual, EqualTo
from datetime import datetime

# Load embeddings table
emb_table = catalog.load_table("embeddings.transcripts")

# What did we know about AAPL as of Jan 1, 2024?
df = emb_table.scan(
    row_filter=And(
        EqualTo("ticker", "AAPL"),
        LessThanOrEqual("knowledge_ts", datetime(2024, 1, 1).isoformat()),
    ),
    selected_fields=("ticker", "fiscal_period", "chunk_idx", "embedding"),
).to_pandas()

import numpy as np
E = np.stack(df["embedding"].values)
print(f"Loaded {len(E)} embeddings with shape {E.shape}")