{
  "id": "stack-profile:data-pipeline-orchestration",
  "_kind": "StackProfile",
  "_file": "domain/stack-profiles/deep-stacks-7.yaml",
  "_cluster": "domain",
  "attributes": {
    "displayName": "Data Pipeline Orchestration (Python, Airflow, dbt, PostgreSQL, Docker)",
    "description": "A data pipeline orchestration platform built around Apache Airflow for\nworkflow scheduling and dbt for SQL-based data transformations, creating\na modern ELT stack where raw data lands in PostgreSQL and is progressively\nrefined through dbt models into analytics-ready tables. Airflow DAGs\ncoordinate extraction from source systems, dbt model runs, data quality\nchecks, and downstream notifications. Python scripts handle custom\nextraction logic and API integrations. SQLAlchemy provides programmatic\ndatabase access for pipeline metadata. Docker Compose runs the complete\nAirflow cluster (scheduler, webserver, workers) alongside PostgreSQL\nfor local development. The tradeoff is Airflow's operational complexity\nand the learning curve of dbt's ref-based dependency graph, but the\ncombination provides unmatched visibility into data lineage.\n",
    "composes": [
      "language:python",
      "tool:airflow",
      "library:sqlalchemy",
      "library:alembic",
      "library:pandas",
      "library:boto3",
      "tool:docker",
      "tool:docker-compose",
      "language:sql"
    ]
  },
  "outgoingEdges": [
    {
      "from": "stack-profile:data-pipeline-orchestration",
      "to": "language:python",
      "kind": "composed_of"
    },
    {
      "from": "stack-profile:data-pipeline-orchestration",
      "to": "tool:airflow",
      "kind": "composed_of"
    },
    {
      "from": "stack-profile:data-pipeline-orchestration",
      "to": "library:sqlalchemy",
      "kind": "composed_of"
    },
    {
      "from": "stack-profile:data-pipeline-orchestration",
      "to": "library:alembic",
      "kind": "composed_of"
    },
    {
      "from": "stack-profile:data-pipeline-orchestration",
      "to": "library:pandas",
      "kind": "composed_of"
    },
    {
      "from": "stack-profile:data-pipeline-orchestration",
      "to": "library:boto3",
      "kind": "composed_of"
    },
    {
      "from": "stack-profile:data-pipeline-orchestration",
      "to": "tool:docker",
      "kind": "composed_of"
    },
    {
      "from": "stack-profile:data-pipeline-orchestration",
      "to": "tool:docker-compose",
      "kind": "composed_of"
    },
    {
      "from": "stack-profile:data-pipeline-orchestration",
      "to": "language:sql",
      "kind": "composed_of"
    },
    {
      "from": "stack-profile:data-pipeline-orchestration",
      "to": "role:data-engineer",
      "kind": "used_by_role"
    },
    {
      "from": "stack-profile:data-pipeline-orchestration",
      "to": "role:analytics-engineer",
      "kind": "used_by_role"
    },
    {
      "from": "stack-profile:data-pipeline-orchestration",
      "to": "workflow:data-pipeline-deployment",
      "kind": "follows_workflow"
    },
    {
      "from": "stack-profile:data-pipeline-orchestration",
      "to": "workflow:data-pipeline-monitoring",
      "kind": "follows_workflow"
    },
    {
      "from": "stack-profile:data-pipeline-orchestration",
      "to": "domain:data-engineering",
      "kind": "applies_to"
    },
    {
      "from": "stack-profile:data-pipeline-orchestration",
      "to": "domain:business-intelligence",
      "kind": "applies_to"
    },
    {
      "from": "stack-profile:data-pipeline-orchestration",
      "to": "skill-area:etl-pipelines",
      "kind": "requires_skill_area"
    },
    {
      "from": "stack-profile:data-pipeline-orchestration",
      "to": "skill-area:python-data-pipelines",
      "kind": "requires_skill_area"
    },
    {
      "from": "stack-profile:data-pipeline-orchestration",
      "to": "skill-area:dbt-modeling",
      "kind": "requires_skill_area"
    },
    {
      "from": "stack-profile:data-pipeline-orchestration",
      "to": "skill-area:data-quality",
      "kind": "requires_skill_area"
    },
    {
      "from": "stack-profile:data-pipeline-orchestration",
      "to": "skill-area:task-scheduling-cron-jobs",
      "kind": "requires_skill_area"
    }
  ],
  "incomingEdges": []
}