Module 15: CI/CD & Deployment

# macOS
brew tap databricks/tap && brew install databricks
# Windows
winget install Databricks.DatabricksCLI
# Linux / WSL
curl -fsSL https://raw.githubusercontent.com/databricks/setup-cli/main/install.sh | sh

databricks -v          # shows the version

databricks auth login --host https://<workspace-url>     # OAuth in the browser (recommended)
export DATABRICKS_HOST=https://<workspace-url>           # token, for scripts and CI
export DATABRICKS_TOKEN=<token>

databricks current-user me
databricks jobs list
databricks workspace list /
databricks fs ls dbfs:/Volumes/<catalog>/dbx_course/raw

# pip install databricks-sdk
from databricks.sdk import WorkspaceClient

w = WorkspaceClient()                         # uses the same auth as the CLI (env vars or profile)

for job in w.jobs.list():
    print(job.job_id, job.settings.name)

run = w.jobs.run_now(job_id=123)              # start a job
result = run.result()                         # wait for it to finish
print(result.state.result_state)

databricks bundle init            # create a project from a template
databricks bundle validate        # check the config
databricks bundle deploy -t dev   # create or update jobs, pipelines
databricks bundle run my_job -t dev
databricks bundle destroy -t dev  # remove what the bundle created

# databricks.yml
bundle:
  name: orders-etl

resources:
  jobs:
    daily_orders:
      name: daily-orders
      tasks:
        - task_key: ingest
          notebook_task:
            notebook_path: ./src/ingest.py
          environment_key: default
      environments:
        - environment_key: default
          spec:
            environment_version: "3"

targets:
  dev:
    mode: development
    default: true
  prod:
    mode: production
    workspace:
      host: https://<prod-workspace-url>

pip install databricks-connect      # use the version that matches your serverless environment

from databricks.connect import DatabricksSession

spark = DatabricksSession.builder.serverless().getOrCreate()
print(spark.range(5).count())

# serverless environment in a bundle
environments:
  - environment_key: default
    spec:
      environment_version: "3"
      dependencies:
        - pandas==2.2.3
        - ./dist/*.whl

# .github/workflows/ci.yml
name: ci
on:
  pull_request:
    types: [opened, synchronize]

jobs:
  test-and-deploy-dev:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with: { python-version: "3.11" }
      - run: pip install -r requirements-dev.txt
      - run: ruff check . && pytest tests/unit
      - uses: databricks/setup-cli@main
      - run: databricks bundle validate -t dev
        env:
          DATABRICKS_HOST: ${{ vars.DATABRICKS_HOST }}
          DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }}
      - run: databricks bundle deploy -t dev
        env:
          DATABRICKS_HOST: ${{ vars.DATABRICKS_HOST }}
          DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }}

# .github/workflows/release.yml
on:
  push:
    branches: [main]
concurrency: deploy-prod            # one deployment at a time
jobs:
  deploy-prod:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: databricks/setup-cli@main
      - run: databricks bundle deploy -t prod
        env:
          DATABRICKS_HOST: ${{ vars.DATABRICKS_HOST }}
          DATABRICKS_TOKEN: ${{ secrets.SP_TOKEN }}

variables:
  catalog:
    description: Target catalog
    default: main
  schema:
    description: Target schema

targets:
  dev:
    mode: development
    default: true
    variables:
      schema: dbx_dev
  prod:
    mode: production
    variables:
      schema: dbx_prod
    run_as:
      service_principal_name: <application-id>
    permissions:
      - group_name: data-engineers
        level: CAN_VIEW

resources:
  jobs:
    daily_orders:
      name: daily-orders
      parameters:
        - name: catalog
          default: ${var.catalog}
        - name: schema
          default: ${var.schema}

terraform {
  required_providers { databricks = { source = "databricks/databricks" } }
}
resource "databricks_job" "daily" {
  name = "daily-orders"
  # ...
}

databricks secrets create-scope my_app
databricks secrets put-secret my_app api_key --string-value "<value>"

api_key = dbutils.secrets.get(scope="my_app", key="api_key")
print(api_key)                      # shows [REDACTED]

git tag -a v1.4.0 -m "Release 1.4.0"
git push origin v1.4.0