Added a tolerance to run on a dedicated node for stacking.

banzai_nres/settings.py

@@ -182,3 +182,4 @@
 SIMBAD_CLASS = os.getenv('BANZAI_SIMBAD', 'astroquery.simbad.Simbad')
 
 CELERY_TASK_QUEUE_NAME = os.getenv('CELERY_TASK_QUEUE_NAME', 'celery')
+CELERY_STACK_QUEUE_NAME = os.getenv('CELERY_STACK_QUEUE_NAME', 'celery')

helm-chart/banzai-nres/templates/_helpers.tpl

@@ -173,6 +173,8 @@ Celery task queue configuration
   value: {{ .Values.banzaiNres.queueName | quote }}
 - name: CELERY_TASK_QUEUE_NAME
   value: {{ .Values.banzaiNres.celeryTaskQueueName | quote }}
+- name: CELERY_STACK_QUEUE_NAME
+  value: {{ .Values.banzaiNres.celeryStackQueueName | quote }}
 - name: BANZAI_WORKER_LOGLEVEL
   value: {{ .Values.banzaiNres.banzaiWorkerLogLevel | quote }}
 - name: PHOENIX_FILE_LOCATION

helm-chart/banzai-nres/templates/stacker.yaml

@@ -0,0 +1,68 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ include "banzai-nres.fullname" . -}} -workers
+  labels:
+{{ include "banzai-nres.labels" . | indent 4 }}
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: {{ include "banzai-nres.name" . }}
+      app.kubernetes.io/instance: {{ .Release.Name }}
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: {{ include "banzai-nres.name" . }}
+        app.kubernetes.io/instance: "{{ .Release.Name }}"
+    spec:
+    {{- with .Values.imagePullSecrets }}
+      imagePullSecrets:
+        {{- toYaml . | nindent 8 }}
+    {{- end }}
+      containers:
+        - name: {{ .Chart.Name }}
+          image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
+          imagePullPolicy: {{ .Values.image.pullPolicy }}
+          securityContext:
+            runAsUser: 10087
+            runAsGroup: 10000
+          command:
+            - "celery"
+            - "-A"
+            - "banzai"
+            - "worker"
+            - "--concurrency"
+            - "1"
+            - "-l"
+            - "info"
+            - "-Q"
+            - "$(CELERY_STACK_QUEUE_NAME)"
+          env:
+            - name: OMP_NUM_THREADS
+              value: "8"
+            {{- include "banzai-nres.Env" . | nindent 12 }}
+          volumeMounts:
+            - name: tmp
+              mountPath: /tmp
+              readOnly: false
+          resources:
+            requests:
+              cpu: "0.75"
+              memory: "1Gi"
+            limits:
+              cpu: "8"
+              memory: "16Gi"
+      volumes:
+      - name: tmp
+        emptyDir:
+          sizeLimit: 100Gi
+    {{- with .Values.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+    {{- with .Values.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+    {{- end }}
+      tolerations:
+          - dedicated: "banzai-disk"

helm-chart/banzai-nres/values-dev.yaml

@@ -36,6 +36,7 @@ banzaiNres:
   queueName: banzai_nres_dev_pipeline
   phoenixFileLocation: s3://banzai-nres-phoenix-models-lco-global
   celeryTaskQueueName: banzai_nres
+  celeryStackQueueName: banzai_nres_stack
 
 # CronJob configuration to periodically update instrument table in BANZAI-NRES DB
 instrumentTableCronjob:

helm-chart/banzai-nres/values-prod.yaml

@@ -39,6 +39,7 @@ banzaiNres:
   queueName: banzai_nres_pipeline
   phoenixFileLocation: s3://banzai-nres-phoenix-models-lco-global
   celeryTaskQueueName: banzai_nres
+  celeryStackQueueName: banzai_nres_stack
 
 # CronJob configuration to periodically update instrument table in BANZAI DB
 instrumentTableCronjob: