WIP: k8s lineages

.dockerignore

@@ -6,6 +6,8 @@
 .roar
 .ralph
 dist
+# roar-runtime image build needs the packaged wheel
+!dist/roar_cli-*.whl
 __pycache__
 **/__pycache__
 *.pyc

.gitignore

@@ -230,3 +230,6 @@ tests/benchmarks/results/
 # Design docs — internal only, not for version control
 docs/design/
 scratch/
+
+# k8s e2e harness downloaded tooling
+tests/backends/k8s/.tools/

deploy/charts/roar-lineage-webhook/Chart.yaml

@@ -0,0 +1,14 @@
+apiVersion: v2
+name: roar-lineage-webhook
+description: >-
+  Zero-touch ML training lineage for Kubernetes: a mutating admission
+  webhook that instruments Jobs, JobSets, PyTorchJobs, TrainJobs, and
+  RayJobs in opted-in namespaces with roar lineage capture.
+type: application
+version: 0.1.0
+appVersion: "0.3.7"
+keywords:
+  - lineage
+  - provenance
+  - ml
+  - admission-webhook

deploy/charts/roar-lineage-webhook/templates/_helpers.tpl

@@ -0,0 +1,30 @@
+{{- define "roar-webhook.fullname" -}}
+{{- printf "%s-roar-lineage-webhook" .Release.Name | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{- define "roar-webhook.labels" -}}
+app.kubernetes.io/name: roar-lineage-webhook
+app.kubernetes.io/instance: {{ .Release.Name }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end -}}
+
+{{- define "roar-webhook.selectorLabels" -}}
+app.kubernetes.io/name: roar-lineage-webhook
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end -}}
+
+{{- define "roar-webhook.runtimeImage" -}}
+{{- if .Values.injection.runtimeImage -}}
+{{ .Values.injection.runtimeImage }}
+{{- else -}}
+{{ .Values.image.repository }}:{{ .Values.image.tag }}
+{{- end -}}
+{{- end -}}
+
+{{- define "roar-webhook.tlsSecretName" -}}
+{{- if .Values.tls.secretName -}}
+{{ .Values.tls.secretName }}
+{{- else -}}
+{{ include "roar-webhook.fullname" . }}-tls
+{{- end -}}
+{{- end -}}

deploy/charts/roar-lineage-webhook/templates/certificate.yaml

@@ -0,0 +1,26 @@
+{{- if .Values.certManager.enabled }}
+apiVersion: cert-manager.io/v1
+kind: Issuer
+metadata:
+  name: {{ include "roar-webhook.fullname" . }}-selfsigned
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "roar-webhook.labels" . | nindent 4 }}
+spec:
+  selfSigned: {}
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: {{ include "roar-webhook.fullname" . }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "roar-webhook.labels" . | nindent 4 }}
+spec:
+  secretName: {{ include "roar-webhook.tlsSecretName" . }}
+  dnsNames:
+    - {{ include "roar-webhook.fullname" . }}.{{ .Release.Namespace }}.svc
+    - {{ include "roar-webhook.fullname" . }}.{{ .Release.Namespace }}.svc.cluster.local
+  issuerRef:
+    name: {{ include "roar-webhook.fullname" . }}-selfsigned
+{{- end }}

deploy/charts/roar-lineage-webhook/templates/deployment.yaml

@@ -0,0 +1,95 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ include "roar-webhook.fullname" . }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "roar-webhook.labels" . | nindent 4 }}
+spec:
+  replicas: {{ .Values.replicas }}
+  selector:
+    matchLabels:
+      {{- include "roar-webhook.selectorLabels" . | nindent 6 }}
+  template:
+    metadata:
+      labels:
+        {{- include "roar-webhook.selectorLabels" . | nindent 8 }}
+    spec:
+      serviceAccountName: {{ include "roar-webhook.fullname" . }}
+      containers:
+        - name: webhook
+          image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
+          imagePullPolicy: {{ .Values.image.pullPolicy }}
+          command:
+            - python
+            - -m
+            - roar.backends.k8s.webhook
+            - --port=8443
+            - --cert=/tls/tls.crt
+            - --key=/tls/tls.key
+          env:
+            - name: ROAR_WEBHOOK_GLAAS_URL
+              value: {{ required "glaas.url is required" .Values.glaas.url | quote }}
+            - name: ROAR_WEBHOOK_CLUSTER_GLAAS_URL
+              value: {{ required "glaas.clusterUrl is required" .Values.glaas.clusterUrl | quote }}
+            - name: ROAR_WEBHOOK_TRACER
+              value: {{ .Values.injection.tracer | quote }}
+            - name: ROAR_WEBHOOK_RUNTIME_SOURCE
+              value: {{ .Values.injection.runtimeSource | quote }}
+            - name: ROAR_WEBHOOK_RUNTIME_IMAGE
+              value: {{ include "roar-webhook.runtimeImage" . | quote }}
+            - name: ROAR_WEBHOOK_RUNTIME_REQUIREMENT
+              value: {{ .Values.injection.runtimeRequirement | quote }}
+            - name: ROAR_WEBHOOK_SESSION_TTL
+              value: {{ .Values.injection.sessionTtlSeconds | quote }}
+            {{- if .Values.injection.proxySidecar }}
+            - name: ROAR_WEBHOOK_PROXY_SIDECAR
+              value: "true"
+            - name: ROAR_WEBHOOK_PROXY_UPSTREAM
+              value: {{ .Values.injection.proxyUpstream | quote }}
+            {{- end }}
+            - name: ROAR_NO_TELEMETRY
+              value: "1"
+          ports:
+            - containerPort: 8443
+          volumeMounts:
+            - name: tls
+              mountPath: /tls
+              readOnly: true
+          readinessProbe:
+            httpGet:
+              path: /healthz
+              port: 8443
+              scheme: HTTPS
+            initialDelaySeconds: 2
+            periodSeconds: 3
+          {{- with .Values.resources }}
+          resources:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+      {{- with .Values.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      volumes:
+        - name: tls
+          secret:
+            secretName: {{ include "roar-webhook.tlsSecretName" . }}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "roar-webhook.fullname" . }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "roar-webhook.labels" . | nindent 4 }}
+spec:
+  selector:
+    {{- include "roar-webhook.selectorLabels" . | nindent 4 }}
+  ports:
+    - port: 443
+      targetPort: 8443

deploy/charts/roar-lineage-webhook/templates/rbac.yaml

@@ -0,0 +1,34 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ include "roar-webhook.fullname" . }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "roar-webhook.labels" . | nindent 4 }}
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: {{ include "roar-webhook.fullname" . }}-secrets
+  labels:
+    {{- include "roar-webhook.labels" . | nindent 4 }}
+rules:
+  # Fragment-session credential Secrets created in opted-in namespaces.
+  - apiGroups: [""]
+    resources: ["secrets"]
+    verbs: ["create"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: {{ include "roar-webhook.fullname" . }}-secrets
+  labels:
+    {{- include "roar-webhook.labels" . | nindent 4 }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: {{ include "roar-webhook.fullname" . }}-secrets
+subjects:
+  - kind: ServiceAccount
+    name: {{ include "roar-webhook.fullname" . }}
+    namespace: {{ .Release.Namespace }}

deploy/charts/roar-lineage-webhook/templates/secret-cleanup.yaml

@@ -0,0 +1,70 @@
+{{- if .Values.secretCleanup.enabled }}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ include "roar-webhook.fullname" . }}-secret-cleanup
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "roar-webhook.labels" . | nindent 4 }}
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: {{ include "roar-webhook.fullname" . }}-secret-cleanup
+  labels:
+    {{- include "roar-webhook.labels" . | nindent 4 }}
+rules:
+  - apiGroups: [""]
+    resources: ["secrets"]
+    verbs: ["list", "delete"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: {{ include "roar-webhook.fullname" . }}-secret-cleanup
+  labels:
+    {{- include "roar-webhook.labels" . | nindent 4 }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: {{ include "roar-webhook.fullname" . }}-secret-cleanup
+subjects:
+  - kind: ServiceAccount
+    name: {{ include "roar-webhook.fullname" . }}-secret-cleanup
+    namespace: {{ .Release.Namespace }}
+---
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: {{ include "roar-webhook.fullname" . }}-secret-cleanup
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "roar-webhook.labels" . | nindent 4 }}
+spec:
+  schedule: {{ .Values.secretCleanup.schedule | quote }}
+  concurrencyPolicy: Forbid
+  successfulJobsHistoryLimit: 1
+  failedJobsHistoryLimit: 3
+  jobTemplate:
+    spec:
+      backoffLimit: 1
+      ttlSecondsAfterFinished: 86400
+      template:
+        metadata:
+          labels:
+            {{- include "roar-webhook.selectorLabels" . | nindent 12 }}
+        spec:
+          serviceAccountName: {{ include "roar-webhook.fullname" . }}-secret-cleanup
+          restartPolicy: Never
+          containers:
+            - name: cleanup
+              image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
+              imagePullPolicy: {{ .Values.image.pullPolicy }}
+              env:
+                - name: ROAR_SECRET_MAX_AGE_SECONDS
+                  value: {{ .Values.secretCleanup.maxAgeSeconds | quote }}
+              command:
+                - python
+                - -m
+                - roar.backends.k8s.secret_cleanup
+{{- end }}

deploy/charts/roar-lineage-webhook/templates/webhook.yaml

@@ -0,0 +1,49 @@
+apiVersion: admissionregistration.k8s.io/v1
+kind: MutatingWebhookConfiguration
+metadata:
+  name: {{ include "roar-webhook.fullname" . }}
+  labels:
+    {{- include "roar-webhook.labels" . | nindent 4 }}
+  {{- if .Values.certManager.enabled }}
+  annotations:
+    cert-manager.io/inject-ca-from: {{ .Release.Namespace }}/{{ include "roar-webhook.fullname" . }}
+  {{- end }}
+webhooks:
+  - name: lineage.roar.glaas.ai
+    admissionReviewVersions: ["v1"]
+    sideEffects: NoneOnDryRun
+    failurePolicy: {{ .Values.webhook.failurePolicy }}
+    reinvocationPolicy: {{ .Values.webhook.reinvocationPolicy }}
+    timeoutSeconds: {{ .Values.webhook.timeoutSeconds }}
+    namespaceSelector:
+      matchLabels:
+        {{ .Values.namespaceSelector.key }}: {{ .Values.namespaceSelector.value }}
+    clientConfig:
+      service:
+        name: {{ include "roar-webhook.fullname" . }}
+        namespace: {{ .Release.Namespace }}
+        path: /mutate
+      {{- if not .Values.certManager.enabled }}
+      caBundle: {{ required "tls.caBundle is required unless certManager.enabled" .Values.tls.caBundle }}
+      {{- end }}
+    rules:
+      - apiGroups: ["batch"]
+        apiVersions: ["v1"]
+        operations: ["CREATE"]
+        resources: ["jobs"]
+      - apiGroups: ["jobset.x-k8s.io"]
+        apiVersions: ["*"]
+        operations: ["CREATE"]
+        resources: ["jobsets"]
+      - apiGroups: ["kubeflow.org"]
+        apiVersions: ["v1"]
+        operations: ["CREATE"]
+        resources: ["pytorchjobs"]
+      - apiGroups: ["trainer.kubeflow.org"]
+        apiVersions: ["*"]
+        operations: ["CREATE"]
+        resources: ["trainjobs"]
+      - apiGroups: ["ray.io"]
+        apiVersions: ["v1"]
+        operations: ["CREATE"]
+        resources: ["rayjobs"]