Module 18: Cost & Resource Optimization

# an illustration with made-up numbers
dbu_per_hour = 4              # depends on the compute size
price_per_dbu = 0.50          # placeholder: SKU and plan decide this
hours = 2.5

databricks_cost = dbu_per_hour * price_per_dbu * hours
print(round(databricks_cost, 2))     # 5.0 (plus VM cost if classic compute)

{
  "autotermination_minutes": { "type": "fixed", "value": 30 },
  "autoscale.max_workers":   { "type": "range", "maxValue": 8 },
  "node_type_id":            { "type": "allowlist", "values": ["m5.xlarge", "m5.2xlarge"] },
  "custom_tags.team":        { "type": "unlimited" },
  "dbus_per_hour":           { "type": "range", "maxValue": 20 }
}

# decide with a measurement, not a guess:
#   run the same job twice (with and without Photon), record runtime and DBUs
#   cost_per_run = dbus_per_hour * price_per_dbu * runtime_hours

%sql
DESCRIBE DETAIL orders_delta

# table sizes in the schema (views are skipped)
import pyspark.sql.functions as F

rows = []
for t in spark.sql("SHOW TABLES").collect():
    if t.isTemporary:
        continue
    try:
        d = spark.sql(f"DESCRIBE DETAIL {t.tableName}").first()
        rows.append((t.tableName, d["numFiles"], d["sizeInBytes"]))
    except Exception:
        pass                      # views and non-Delta tables have no detail

df = spark.createDataFrame(rows, "table STRING, files LONG, bytes LONG")
display(df.orderBy(F.desc("bytes")))

%sql
-- cost units per team tag (system tables: may not exist on Free Edition)
SELECT custom_tags['team'] AS team, usage_date, sum(usage_quantity) AS dbus
FROM system.billing.usage
GROUP BY ALL
ORDER BY usage_date DESC