feat: initial datalake and stats site (#28666)

2026-05-25 17:34:04 -05:00
parent 633b5d6208
commit 5b02ac4d33
68 changed files with 8967 additions and 42 deletions
@@ -0,0 +1,156 @@
+import { bigint, char, datetime, decimal, index, int, mysqlTable, uniqueIndex, varchar } from "drizzle-orm/mysql-core"
+
+export const modelStat = mysqlTable(
+  "model_stat",
+  {
+    ...periodColumns(),
+    provider: varchar({ length: 128 }).notNull(),
+    model: varchar({ length: 256 }).notNull(),
+    provider_model: varchar({ length: 256 }).notNull().default(""),
+    ...metricColumns(),
+    rank_by_tokens: int(),
+    rank_by_requests: int(),
+    rank_by_cost: int(),
+    ...timestampColumns(),
+  },
+  (table) => [
+    uniqueIndex("uniq_model_period").on(
+      table.grain,
+      table.period_start,
+      table.dataset,
+      table.tier,
+      table.client,
+      table.source,
+      table.provider,
+      table.model,
+    ),
+    index("idx_leaderboard_tokens").on(table.grain, table.period_start, table.dataset, table.tier, table.total_tokens),
+    index("idx_model").on(table.model, table.grain, table.period_start),
+  ],
+)
+
+export const providerStat = mysqlTable(
+  "provider_stat",
+  {
+    ...periodColumns(),
+    provider: varchar({ length: 128 }).notNull(),
+    ...metricColumns(),
+    ...marketShareColumns(),
+    rank_by_tokens: int(),
+    rank_by_requests: int(),
+    rank_by_sessions: int(),
+    rank_by_cost: int(),
+    ...timestampColumns(),
+  },
+  (table) => [
+    uniqueIndex("uniq_provider_period").on(
+      table.grain,
+      table.period_start,
+      table.dataset,
+      table.tier,
+      table.client,
+      table.source,
+      table.provider,
+    ),
+    index("idx_provider_leaderboard_tokens").on(
+      table.grain,
+      table.period_start,
+      table.dataset,
+      table.tier,
+      table.total_tokens,
+    ),
+    index("idx_provider_market_share").on(
+      table.grain,
+      table.period_start,
+      table.dataset,
+      table.tier,
+      table.market_share_tokens,
+    ),
+    index("idx_provider_rank").on(table.grain, table.period_start, table.dataset, table.tier, table.rank_by_tokens),
+    index("idx_provider").on(table.provider, table.grain, table.period_start),
+  ],
+)
+
+export const geoStat = mysqlTable(
+  "geo_stat",
+  {
+    ...periodColumns(),
+    country: char({ length: 2 }).notNull(),
+    continent: varchar({ length: 8 }).notNull().default(""),
+    ...metricColumns(),
+    ...marketShareColumns(),
+    rank_by_tokens: int(),
+    rank_by_requests: int(),
+    rank_by_sessions: int(),
+    rank_by_cost: int(),
+    ...timestampColumns(),
+  },
+  (table) => [
+    uniqueIndex("uniq_country_period").on(
+      table.grain,
+      table.period_start,
+      table.dataset,
+      table.tier,
+      table.client,
+      table.source,
+      table.country,
+    ),
+    index("idx_country_map_tokens").on(table.grain, table.period_start, table.dataset, table.tier, table.total_tokens),
+    index("idx_country_rank").on(table.grain, table.period_start, table.dataset, table.tier, table.rank_by_tokens),
+    index("idx_country").on(table.country, table.grain, table.period_start),
+    index("idx_continent").on(table.continent, table.grain, table.period_start),
+  ],
+)
+
+function periodColumns() {
+  return {
+    id: bigint({ mode: "number" }).autoincrement().primaryKey(),
+    grain: varchar({ length: 16 }).notNull(),
+    period_start: datetime({ mode: "date" }).notNull(),
+    period_end: datetime({ mode: "date" }).notNull(),
+    dataset: varchar({ length: 64 }).notNull().default("all"),
+    tier: varchar({ length: 64 }).notNull().default("all"),
+    client: varchar({ length: 64 }).notNull().default("all"),
+    source: varchar({ length: 64 }).notNull().default("all"),
+  }
+}
+
+function metricColumns() {
+  return {
+    sessions: bigint({ mode: "number" }).notNull().default(0),
+    requests: bigint({ mode: "number" }).notNull().default(0),
+    input_tokens: bigint({ mode: "number" }).notNull().default(0),
+    output_tokens: bigint({ mode: "number" }).notNull().default(0),
+    reasoning_tokens: bigint({ mode: "number" }).notNull().default(0),
+    cache_read_tokens: bigint({ mode: "number" }).notNull().default(0),
+    total_tokens: bigint({ mode: "number" }).notNull().default(0),
+    input_cost_microcents: bigint({ mode: "number" }).notNull().default(0),
+    output_cost_microcents: bigint({ mode: "number" }).notNull().default(0),
+    total_cost_microcents: bigint({ mode: "number" }).notNull().default(0),
+    avg_duration_ms: decimal({ precision: 12, scale: 2, mode: "number" }),
+    p50_duration_ms: int(),
+    p95_duration_ms: int(),
+    avg_ttfb_ms: decimal({ precision: 12, scale: 2, mode: "number" }),
+    p50_ttfb_ms: int(),
+    p95_ttfb_ms: int(),
+    avg_output_tps: decimal({ precision: 12, scale: 4, mode: "number" }),
+    success_count: bigint({ mode: "number" }).notNull().default(0),
+    error_count: bigint({ mode: "number" }).notNull().default(0),
+    sample_count: bigint({ mode: "number" }).notNull().default(0),
+  }
+}
+
+function marketShareColumns() {
+  return {
+    market_share_tokens: decimal({ precision: 10, scale: 6, mode: "number" }),
+    market_share_requests: decimal({ precision: 10, scale: 6, mode: "number" }),
+    market_share_sessions: decimal({ precision: 10, scale: 6, mode: "number" }),
+  }
+}
+
+function timestampColumns() {
+  return {
+    created_at: datetime({ mode: "date" }).notNull().defaultNow(),
+    updated_at: datetime({ mode: "date" }).notNull().defaultNow().onUpdateNow(),
+  }
+}