Connect Databricks to VectorFin Financial Iceberg Data

-- Create storage credential (service account JSON from VectorFin)
CREATE STORAGE CREDENTIAL vf_gcs_cred
  WITH GCS SERVICE ACCOUNT KEY = '<base64-encoded-service-account-json>';

-- Create external location
CREATE EXTERNAL LOCATION vf_warehouse
  URL 'gs://vectorfinancials-data/warehouse'
  WITH (CREDENTIAL vf_gcs_cred);

-- Validate access
VALIDATE STORAGE CREDENTIAL vf_gcs_cred;

-- Create a catalog for VectorFin data
CREATE CATALOG IF NOT EXISTS vectorfin;
CREATE SCHEMA IF NOT EXISTS vectorfin.embeddings;
CREATE SCHEMA IF NOT EXISTS vectorfin.signals;

-- Register transcript embeddings table
CREATE TABLE IF NOT EXISTS vectorfin.embeddings.transcripts
USING ICEBERG
LOCATION 'gs://vectorfinancials-data/warehouse/embeddings/transcripts/';

-- Register signals tables
CREATE TABLE IF NOT EXISTS vectorfin.signals.whystock_score
USING ICEBERG
LOCATION 'gs://vectorfinancials-data/warehouse/signals/whystock_score/';

CREATE TABLE IF NOT EXISTS vectorfin.signals.volatility
USING ICEBERG
LOCATION 'gs://vectorfinancials-data/warehouse/signals/volatility/';

# Query transcript embeddings
df = spark.table("vectorfin.embeddings.transcripts")
aapl_df = df.filter(df.ticker == "AAPL").filter(df.fiscal_period.startswith("2024"))
display(aapl_df.select("ticker", "fiscal_period", "chunk_idx").limit(20))

# Load quant signals
signals = spark.table("vectorfin.signals.whystock_score")
top_signals = signals.filter(signals.date >= "2024-01-01") \
  .orderBy(signals.score.desc()) \
  .limit(50)
display(top_signals)

import numpy as np
from pyspark.sql.functions import col

# Load embeddings as numpy arrays
df = spark.table("vectorfin.embeddings.transcripts") \
  .filter(col("ticker") == "NVDA") \
  .select("fiscal_period", "chunk_idx", "embedding") \
  .toPandas()

embeddings = np.stack(df["embedding"].values)

# Your query vector (e.g., from Gemini text-embedding-004)
query_vec = np.array([...])  # 768-dim

# Cosine similarity
scores = embeddings @ query_vec / (np.linalg.norm(embeddings, axis=1) * np.linalg.norm(query_vec))
top_k = df.iloc[scores.argsort()[::-1][:5]]
print(top_k[["fiscal_period", "chunk_idx"]].to_string())