{
  "id": "lib-process:data-engineering-analytics--etl-elt-pipeline",
  "_kind": "LibraryProcess",
  "_file": "generated-library/processes.yaml",
  "_cluster": "generated-library",
  "attributes": {
    "displayName": "specializations/data-engineering-analytics/etl-elt-pipeline",
    "description": "ETL/ELT Pipeline Setup - Design and implement a comprehensive data pipeline from source to destination,\nincluding source connection, ingestion layer, transformation logic, data quality gates, orchestration, and monitoring.\nSupports both batch and streaming data patterns with comprehensive validation and error handling.",
    "libraryPath": "library/specializations/data-engineering-analytics/etl-elt-pipeline.js",
    "specialization": "data-engineering-analytics",
    "references": [
      "- ETL Best Practices: https://docs.airbyte.com/understanding-airbyte/",
      "- Data Quality: https://www.datakitchen.io/data-quality-fundamentals",
      "- Apache Airflow: https://airflow.apache.org/docs/",
      "- Dagster: https://docs.dagster.io/",
      "- dbt (data build tool): https://docs.getdbt.com/",
      "- Great Expectations: https://greatexpectations.io/",
      "- Data Engineering Best Practices: https://github.com/DataTalksClub/data-engineering-zoomcamp"
    ],
    "example": "const result = await orchestrate('specializations/data-engineering-analytics/etl-elt-pipeline', {\n  pipelineName: 'Customer Analytics Pipeline',\n  sources: [\n    { type: 'postgresql', name: 'transactional-db', connection: 'prod-db' },\n    { type: 'kafka', name: 'events-stream', topics: ['user-events', 'product-views'] },\n    { type: 's3', name: 'logs-bucket', path: 's3://logs/clickstream/' }\n  ],\n  destinations: [\n    { type: 'snowflake', name: 'analytics-warehouse', schema: 'customer_analytics' },\n    { type: 'redshift', name: 'reporting-db', schema: 'reports' }\n  ],\n  pipelineType: 'hybrid', // 'batch', 'streaming', or 'hybrid'\n  transformationLogic: {\n    stages: ['extract', 'clean', 'enrich', 'aggregate', 'load'],\n    dbtModels: true,\n    customTransformations: ['user-segmentation', 'revenue-attribution']\n  },\n  dataQualityRules: {\n    schemaValidation: true,\n    nullChecks: true,\n    rangeChecks: true,\n    referentialIntegrity: true,\n    customRules: ['email-format', 'positive-revenue']\n  },\n  orchestration: {\n    tool: 'airflow', // 'airflow', 'dagster', 'prefect', 'step-functions'\n    schedule: 'hourly',\n    retryPolicy: { maxRetries: 3, backoff: 'exponential' }\n  }\n});",
    "usesAgents": [
      "data-architect",
      "data-integration-engineer",
      "data-warehouse-engineer",
      "ingestion-engineer",
      "data-platform-engineer",
      "analytics-engineer",
      "data-quality-engineer",
      "pipeline-quality-engineer",
      "reliability-engineer",
      "orchestration-engineer",
      "data-optimization-engineer",
      "observability-engineer",
      "metadata-engineer",
      "performance-engineer",
      "security-engineer",
      "qa-engineer",
      "technical-writer",
      "validation-engineer",
      "deployment-engineer",
      "data-engineering-lead"
    ]
  },
  "outgoingEdges": [
    {
      "from": "lib-process:data-engineering-analytics--etl-elt-pipeline",
      "to": "domain:data-engineering",
      "kind": "lib_applies_to_domain",
      "attributes": {
        "weight": 1
      }
    },
    {
      "from": "lib-process:data-engineering-analytics--etl-elt-pipeline",
      "to": "workflow:data-pipeline-deployment",
      "kind": "lib_implements_workflow",
      "attributes": {
        "weight": 1
      }
    },
    {
      "from": "lib-process:data-engineering-analytics--etl-elt-pipeline",
      "to": "workflow:data-backfill-procedure",
      "kind": "lib_implements_workflow",
      "attributes": {
        "weight": 0.7
      }
    },
    {
      "from": "lib-process:data-engineering-analytics--etl-elt-pipeline",
      "to": "specialization:data-engineering-analytics",
      "kind": "lib_belongs_to_specialization",
      "attributes": {
        "weight": 0.9
      }
    },
    {
      "from": "lib-process:data-engineering-analytics--etl-elt-pipeline",
      "to": "lib-agent:software-architecture--data-architect",
      "kind": "uses_agent",
      "attributes": {
        "weight": 0.8
      }
    },
    {
      "from": "lib-process:data-engineering-analytics--etl-elt-pipeline",
      "to": "lib-agent:game-development--analytics-engineer",
      "kind": "uses_agent",
      "attributes": {
        "weight": 0.8
      }
    },
    {
      "from": "lib-process:data-engineering-analytics--etl-elt-pipeline",
      "to": "lib-agent:data-engineering-analytics--data-quality-engineer",
      "kind": "uses_agent",
      "attributes": {
        "weight": 0.8
      }
    },
    {
      "from": "lib-process:data-engineering-analytics--etl-elt-pipeline",
      "to": "lib-agent:electrical-engineering--reliability-engineer",
      "kind": "uses_agent",
      "attributes": {
        "weight": 0.8
      }
    },
    {
      "from": "lib-process:data-engineering-analytics--etl-elt-pipeline",
      "to": "lib-agent:ai-agents-conversational--observability-engineer",
      "kind": "uses_agent",
      "attributes": {
        "weight": 0.8
      }
    },
    {
      "from": "lib-process:data-engineering-analytics--etl-elt-pipeline",
      "to": "lib-agent:software-architecture--performance-engineer",
      "kind": "uses_agent",
      "attributes": {
        "weight": 0.8
      }
    },
    {
      "from": "lib-process:data-engineering-analytics--etl-elt-pipeline",
      "to": "lib-agent:shared--qa-engineer",
      "kind": "uses_agent",
      "attributes": {
        "weight": 0.8
      }
    },
    {
      "from": "lib-process:data-engineering-analytics--etl-elt-pipeline",
      "to": "lib-agent:meta--technical-writer",
      "kind": "uses_agent",
      "attributes": {
        "weight": 0.8
      }
    },
    {
      "from": "lib-process:data-engineering-analytics--etl-elt-pipeline",
      "to": "lib-agent:shared--deployment-engineer",
      "kind": "uses_agent",
      "attributes": {
        "weight": 0.8
      }
    }
  ],
  "incomingEdges": []
}