{
  "id": "lib-process:data-engineering-analytics--streaming-pipeline",
  "_kind": "LibraryProcess",
  "_file": "generated-library/processes.yaml",
  "_cluster": "generated-library",
  "attributes": {
    "displayName": "streaming-pipeline",
    "description": "Streaming Data Pipeline Setup - Complete workflow for designing and implementing\nproduction-ready streaming data pipelines with Kafka/Kinesis setup, stream processing frameworks,\nwindowing, state management, and comprehensive monitoring.",
    "libraryPath": "library/specializations/data-engineering-analytics/streaming-pipeline.js",
    "specialization": "data-engineering-analytics",
    "references": [
      "- Apache Kafka Documentation: https://kafka.apache.org/documentation/\n- AWS Kinesis: https://docs.aws.amazon.com/kinesis/\n- Apache Flink: https://flink.apache.org/\n- Apache Spark Streaming: https://spark.apache.org/streaming/\n- Kafka Streams: https://kafka.apache.org/documentation/streams/\n- Stream Processing Patterns: https://www.confluent.io/blog/event-streaming-patterns/\n- State Management in Flink: https://nightlies.apache.org/flink/flink-docs-stable/docs/dev/datastream/fault-tolerance/state/\n- Exactly-Once Semantics: https://www.confluent.io/blog/exactly-once-semantics-are-possible-heres-how-apache-kafka-does-it/"
    ],
    "example": "const result = await orchestrate('specializations/data-engineering-analytics/streaming-pipeline', {\n  projectName: 'Real-time Analytics Pipeline',\n  streamingPlatform: 'kafka',\n  processingFramework: 'flink',\n  requirements: {\n    throughput: '100000 events/sec',\n    latency: 'sub-second',\n    dataRetention: '7 days',\n    stateBackend: 'rocksdb',\n    monitoring: true,\n    schemas: true,\n    exactlyOnce: true\n  }\n});",
    "usesAgents": [
      "streaming-architect",
      "messaging-engineer",
      "schema-engineer",
      "processing-engineer",
      "state-management-specialist",
      "windowing-specialist",
      "connector-engineer",
      "sink-engineer",
      "backpressure-specialist",
      "monitoring-engineer",
      "lag-monitoring-specialist",
      "performance-engineer",
      "alerting-engineer",
      "autoscaling-engineer",
      "dr-specialist",
      "pipeline-validator",
      "technical-writer"
    ]
  },
  "outgoingEdges": [
    {
      "from": "lib-process:data-engineering-analytics--streaming-pipeline",
      "to": "domain:data-engineering",
      "kind": "lib_applies_to_domain",
      "attributes": {
        "weight": 1
      }
    },
    {
      "from": "lib-process:data-engineering-analytics--streaming-pipeline",
      "to": "workflow:data-pipeline-deployment",
      "kind": "lib_implements_workflow",
      "attributes": {
        "weight": 1
      }
    },
    {
      "from": "lib-process:data-engineering-analytics--streaming-pipeline",
      "to": "specialization:data-engineering-analytics",
      "kind": "lib_belongs_to_specialization",
      "attributes": {
        "weight": 0.9
      }
    },
    {
      "from": "lib-process:data-engineering-analytics--streaming-pipeline",
      "to": "lib-agent:software-architecture--performance-engineer",
      "kind": "uses_agent",
      "attributes": {
        "weight": 0.8
      }
    },
    {
      "from": "lib-process:data-engineering-analytics--streaming-pipeline",
      "to": "lib-agent:devops-sre-platform--dr-specialist",
      "kind": "uses_agent",
      "attributes": {
        "weight": 0.8
      }
    },
    {
      "from": "lib-process:data-engineering-analytics--streaming-pipeline",
      "to": "lib-agent:meta--technical-writer",
      "kind": "uses_agent",
      "attributes": {
        "weight": 0.8
      }
    }
  ],
  "incomingEdges": []
}