feat: add openlineage demo

demos/demos-v2.yaml

@@ -314,3 +314,25 @@ demos:
       - name: branch
         description: Git branch to download demo files from
         default: main
+  openlineage:
+    description: Showcase OpenLineage data lineage from Trino, Airflow and Spark over a shared Iceberg lakehouse, visualized in Marquez
+    documentation: https://docs.stackable.tech/home/stable/demos/openlineage
+    stackableStack: openlineage
+    labels:
+      - trino
+      - airflow
+      - spark
+      - openlineage
+      - marquez
+      - data-lineage
+      - iceberg
+      - garage
+      - s3
+    manifests:
+      - plainYaml: demos/openlineage/rbac.yaml
+      - plainYaml: demos/openlineage/enable-and-run-dag.yaml
+    supportedNamespaces: []
+    resourceRequests:
+      cpu: 10500m
+      memory: 22Gi
+      pvc: 40Gi # 10Gi Garage + 3x8Gi PostgreSQL; data is tiny (tpch.tiny)

demos/openlineage/enable-and-run-dag.yaml

@@ -0,0 +1,35 @@
+---
+# Waits for Airflow to be ready, then un-pauses and triggers the `openlineage_demo` DAG via the
+# Airflow REST API. Running the DAG is what makes Trino execute the queries, so this is what
+# populates Marquez with lineage.
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: enable-and-run-openlineage-dag
+spec:
+  template:
+    spec:
+      containers:
+        - name: enable-and-run-openlineage-dag
+          image: oci.stackable.tech/sdp/tools:1.0.0-stackable0.0.0-dev
+          command:
+            - bash
+            - -euo
+            - pipefail
+            - -c
+            - |
+              kubectl rollout status --watch statefulset/airflow-webserver-default
+              kubectl rollout status --watch statefulset/airflow-scheduler-default
+              AIRFLOW_ADMIN_PASSWORD=$(cat /airflow-credentials/adminUser.password)
+              ACCESS_TOKEN=$(curl -XPOST http://airflow-webserver-default-headless:8080/auth/token -H 'Content-Type: application/json' -d '{"username": "admin", "password": "'$AIRFLOW_ADMIN_PASSWORD'"}' | jq -r .access_token)
+              curl -H "Authorization: Bearer $ACCESS_TOKEN" -H 'Content-Type: application/json' -XPATCH http://airflow-webserver-default-headless:8080/api/v2/dags/openlineage_demo -d '{"is_paused": false}' | jq
+              curl -H "Authorization: Bearer $ACCESS_TOKEN" -H 'Content-Type: application/json' -XPOST http://airflow-webserver-default-headless:8080/api/v2/dags/openlineage_demo/dagRuns -d '{"logical_date": null}' | jq
+          volumeMounts:
+            - name: airflow-credentials
+              mountPath: /airflow-credentials
+      volumes:
+        - name: airflow-credentials
+          secret:
+            secretName: airflow-credentials
+      restartPolicy: OnFailure
+  backoffLimit: 20 # give some time for the Airflow cluster to be available

demos/openlineage/rbac.yaml

@@ -0,0 +1,49 @@
+---
+# Grants two things to every ServiceAccount in the namespace (see the ClusterRoleBinding below):
+#   * the enable-and-run Job waits for the Airflow StatefulSets via `kubectl rollout status`
+#     (pods + statefulsets get/list/watch), and
+#   * the Airflow DAG's inlined SparkKubernetesOperator/Sensor creates and polls the SparkApplication
+#     (sparkapplications create/get/list/watch).
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: openlineage-demo-clusterrole
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - pods
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups:
+      - apps
+    resources:
+      - statefulsets
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups:
+      - spark.stackable.tech
+    resources:
+      - sparkapplications
+    verbs:
+      - get
+      - list
+      - watch
+      - create
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: openlineage-demo-clusterrolebinding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: openlineage-demo-clusterrole
+subjects:
+  - apiGroup: rbac.authorization.k8s.io
+    kind: Group
+    name: system:serviceaccounts

docs/modules/demos/partials/demos.adoc

@@ -8,6 +8,7 @@
 * xref:logging.adoc[]
 * xref:nifi-kafka-druid-earthquake-data.adoc[]
 * xref:nifi-kafka-druid-water-level-data.adoc[]
+* xref:openlineage.adoc[]
 * xref:opensearch-rag.adoc[]
 * xref:signal-processing.adoc[]
 * xref:spark-k8s-anomaly-detection-taxi-data.adoc[]