Share PostgreSQL with Substrate

contrib/cncf/technical-review.md

@@ -325,7 +325,7 @@ Default values can be found in [helm/kagent/values.yaml](https://github.com/kage
 **Additional Configurations:**
 For production use, configure:
 
-- External PostgreSQL connection (set `database.postgres.bundled.enabled=false` and set either `database.postgres.url` or `database.postgres.urlFile`)
+- External PostgreSQL connection (set `database.postgres.bundled.enabled=false` and configure `database.postgres.url` or `database.postgres.secretRef`)
 - LLM API keys via Secrets (`providers.openAI.apiKeySecretRef`)
 - TLS for external LLM connections (`modelConfig.tls`)
 - Resource limits based on workload (`agents.*.resources`)

go/core/internal/database/connect.go

@@ -3,8 +3,6 @@ package database
 import (
 	"context"
 	"fmt"
-	"os"
-	"strings"
 	"time"
 
 	"github.com/jackc/pgx/v5"
@@ -14,9 +12,6 @@ import (
 )
 
 // PostgresConfig holds the connection parameters for a Postgres database.
-// URL must be a resolved connection string — use ResolveURL to resolve from
-// a file path before constructing this config.
-//
 // Pool fields are optional: nil leaves the corresponding pgxpool.Config value
 // from ParseConfig unchanged (pgx library defaults).
 type PostgresConfig struct {
@@ -109,27 +104,3 @@ func retryDBConnection(ctx context.Context, cfg *PostgresConfig) (*pgxpool.Pool,
 		}
 	}
 }
-
-// ResolveURL returns url, unless urlFile is non-empty in which case the URL is
-// read from that file. Used by callers (e.g. the migration runner) that need
-// the resolved connection string before a pool is created.
-func ResolveURL(url, urlFile string) (string, error) {
-	if urlFile != "" {
-		return resolveURLFile(urlFile)
-	}
-	return url, nil
-}
-
-// resolveURLFile reads a database connection URL from a file and returns the
-// trimmed contents. Returns an error if the file cannot be read or is empty.
-func resolveURLFile(path string) (string, error) {
-	content, err := os.ReadFile(path)
-	if err != nil {
-		return "", fmt.Errorf("reading URL file: %w", err)
-	}
-	url := strings.TrimSpace(string(content))
-	if url == "" {
-		return "", fmt.Errorf("URL file %s is empty or contains only whitespace", path)
-	}
-	return url, nil
-}

go/core/internal/database/connect_test.go

@@ -2,8 +2,6 @@ package database
 
 import (
 	"context"
-	"os"
-	"path/filepath"
 	"testing"
 	"time"
 
@@ -53,54 +51,3 @@ func TestApplyPoolConfig(t *testing.T) {
 		assert.Equal(t, 10*time.Minute, config.MaxConnLifetime)
 	})
 }
-
-func TestResolveURLFile(t *testing.T) {
-	tests := []struct {
-		name        string
-		fileContent string
-		wantUrl     string
-		wantErr     bool
-	}{
-		{
-			name:        "reads URL from file",
-			fileContent: "postgres://testuser:testpass@host:5432/testdb",
-			wantUrl:     "postgres://testuser:testpass@host:5432/testdb",
-		},
-		{
-			name:        "trims whitespace and newlines",
-			fileContent: "  postgres://user:pass@host:5432/db\n",
-			wantUrl:     "postgres://user:pass@host:5432/db",
-		},
-		{
-			name:        "empty file returns error",
-			fileContent: "",
-			wantErr:     true,
-		},
-		{
-			name:        "whitespace-only file returns error",
-			fileContent: "  \n\t\n  ",
-			wantErr:     true,
-		},
-	}
-
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			tmpFile := filepath.Join(t.TempDir(), "db-url")
-			err := os.WriteFile(tmpFile, []byte(tt.fileContent), 0600)
-			assert.NoError(t, err)
-
-			url, err := resolveURLFile(tmpFile)
-			if tt.wantErr {
-				assert.Error(t, err)
-				return
-			}
-			assert.NoError(t, err)
-			assert.Equal(t, tt.wantUrl, url)
-		})
-	}
-
-	t.Run("missing file returns error", func(t *testing.T) {
-		_, err := resolveURLFile("/nonexistent/path/db-url")
-		assert.Error(t, err)
-	})
-}

go/core/pkg/app/app.go

@@ -163,10 +163,7 @@ func Run(ctx context.Context, opts Options) error {
 		}
 	}()
 
-	dbURL, err := database.ResolveURL(env("POSTGRES_DATABASE_URL", "postgres://postgres:kagent@kagent-postgresql.kagent.svc.cluster.local:5432/postgres"), os.Getenv("POSTGRES_DATABASE_URL_FILE"))
-	if err != nil {
-		return err
-	}
+	dbURL := env("POSTGRES_DATABASE_URL", "postgres://postgres:kagent@kagent-postgresql.kagent.svc.cluster.local:5432/postgres")
 	vectorEnabled := kagentenv.DatabaseVectorEnabled.Get()
 	// Appended, not merged: the built-in tracks must reach their final version
 	// before a library consumer's tables, which may reference them.

helm/README.md

@@ -21,6 +21,62 @@ helm install kagent ./helm/kagent/ --namespace kagent --set providers.default=an
 helm install kagent ./helm/kagent/ --namespace kagent --set providers.default=azureOpenAI  --set providers.azureOpenAI.apiKey=your-openai-api-key
 ```
 
+### Substrate PostgreSQL
+
+Enabling Substrate uses Kagent's bundled PostgreSQL by default. Kagent and
+Substrate share the database connection but use separate schemas.
+
+```yaml
+substrate:
+  enabled: true
+```
+
+To share an external PostgreSQL connection, configure it once for Kagent:
+
+```yaml
+database:
+  postgres:
+    url: postgresql://user:password@database:5432/kagent
+    bundled:
+      enabled: false
+substrate:
+  enabled: true
+```
+
+To share an existing Secret, configure both charts to reference the same
+name and key:
+
+```yaml
+database:
+  postgres:
+    secretRef:
+      name: shared-postgres
+      key: connectionString
+    bundled:
+      enabled: false
+substrate:
+  enabled: true
+  postgres:
+    connectionStringSecretRef:
+      name: shared-postgres
+      key: connectionString
+```
+
+To give Substrate a separate PostgreSQL connection, disable sharing and set
+the Substrate connection directly:
+
+```yaml
+substrate:
+  enabled: true
+  postgres:
+    connectionString: postgresql://user:password@substrate-db:5432/substrate
+    connectionStringSecretRef:
+      enabled: false
+```
+
+For a separate Secret-backed connection, leave `enabled: false` and set
+`connectionStringSecretRef.name` and `key`.
+
 ### Using Make
 
 ```bash

helm/kagent/templates/NOTES.txt

@@ -64,23 +64,25 @@ DOCUMENTATION:
 {{- end }}
 {{ if .Values.database.postgres.bundled.enabled -}}
 ################################################################################
-{{- if and (eq .Values.database.postgres.url "") (eq .Values.database.postgres.urlFile "") }}
+{{- if and (eq .Values.database.postgres.url "") (not .Values.database.postgres.secretRef.name) }}
 # WARNING: BUNDLED DATABASE IN USE                                             #
 ################################################################################
   The bundled PostgreSQL instance is enabled. It is intended for development and
   evaluation only, not suitable for production use. Data may be lost if the
   pod is restarted or rescheduled.
 
   To use an external database, set:
-    database.postgres.url=<your-connection-string> or database.postgres.urlFile=<path>
+    database.postgres.url=<your-connection-string>
+  or database.postgres.secretRef.name=<secret-name>
 {{- else }}
 # NOTE: BUNDLED DATABASE DEPLOYED BUT NOT IN USE BY CONTROLLER                 #
 ################################################################################
   The bundled PostgreSQL pod is running, but the controller is connected to an
-  external database (database.postgres.url or database.postgres.urlFile is set).
+  external database.
 
-  To connect the controller to the bundled instance instead, unset url/urlFile:
+  To connect the controller to the bundled instance instead, unset the external connection:
     database.postgres.url=""
+    database.postgres.secretRef.name=""
   To stop deploying the bundled pod entirely, set:
     database.postgres.bundled.enabled=false
 {{- end }}

helm/kagent/templates/controller-deployment.yaml

@@ -1,3 +1,10 @@
+{{- $databaseConnectionStringSecretRef := .Values.database.postgres.secretRef | default dict -}}
+{{- if hasKey .Values.database.postgres "urlFile" -}}
+{{- fail "database.postgres.urlFile has been removed; use database.postgres.secretRef.{name,key}" -}}
+{{- end -}}
+{{- if and .Values.database.postgres.url (get $databaseConnectionStringSecretRef "name") -}}
+{{- fail "database.postgres.url and database.postgres.secretRef.name are mutually exclusive" -}}
+{{- end -}}
 apiVersion: apps/v1
 kind: Deployment
 metadata:
@@ -101,9 +108,12 @@ spec:
             - name: AUTH_USER_ID_CLAIM
               value: {{ .Values.controller.auth.userIdClaim | quote }}
             {{- end }}
-            {{- if .Values.database.postgres.urlFile }}
-            - name: POSTGRES_DATABASE_URL_FILE
-              value: {{ .Values.database.postgres.urlFile | quote }}
+            {{- if get $databaseConnectionStringSecretRef "name" }}
+            - name: POSTGRES_DATABASE_URL
+              valueFrom:
+                secretKeyRef:
+                  name: {{ get $databaseConnectionStringSecretRef "name" }}
+                  key: {{ get $databaseConnectionStringSecretRef "key" | default "connectionString" }}
             {{- else if .Values.database.postgres.url }}
             - name: POSTGRES_DATABASE_URL
               value: {{ .Values.database.postgres.url | quote }}
@@ -116,7 +126,7 @@ spec:
             - name: POSTGRES_DATABASE_URL
               value: {{ printf "postgres://kagent:$(POSTGRES_PASSWORD)@%s.%s.svc:5432/kagent?sslmode=disable" (include "kagent.postgresqlServiceName" .) (include "kagent.namespace" .) | quote }}
             {{- else }}
-            {{ fail "No database connection configured. Set database.postgres.url, database.postgres.urlFile, or enable database.postgres.bundled." }}
+            {{ fail "No database connection configured. Set database.postgres.url, database.postgres.secretRef.name, or enable database.postgres.bundled." }}
             {{- end }}
             {{- if include "kagent.controller.metricsEnabled" . }}
             - name: METRICS_BIND_ADDRESS

helm/kagent/templates/postgresql-secret.yaml

@@ -11,3 +11,30 @@ type: Opaque
 data:
   POSTGRES_PASSWORD: {{ "kagent" | b64enc | quote }}
 {{- end }}
+{{- $databaseConnectionStringSecretRef := .Values.database.postgres.secretRef | default dict -}}
+{{- $substratePostgres := get .Values.substrate "postgres" | default dict -}}
+{{- $connectionStringSecretRef := get $substratePostgres "connectionStringSecretRef" | default dict -}}
+{{- if and .Values.substrate.enabled (get $connectionStringSecretRef "enabled") (not (get $connectionStringSecretRef "name")) }}
+{{- $connectionString := "" -}}
+{{- if get $databaseConnectionStringSecretRef "name" -}}
+{{- fail "database.postgres.secretRef cannot be inherited by Substrate; set substrate.postgres.connectionStringSecretRef to the same Secret" -}}
+{{- else if .Values.database.postgres.url -}}
+{{- $connectionString = .Values.database.postgres.url -}}
+{{- else if .Values.database.postgres.bundled.enabled -}}
+{{- $connectionString = printf "postgres://kagent:kagent@%s.%s.svc:5432/kagent?sslmode=disable" (include "kagent.postgresqlServiceName" .) (include "kagent.namespace" .) -}}
+{{- else -}}
+{{- fail "No database connection configured. Set database.postgres.url, substrate.postgres.connectionStringSecretRef.name, or enable database.postgres.bundled." -}}
+{{- end }}
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ get $connectionStringSecretRef "name" | default (include "substrate.fullname" (list "postgres-connection" .)) }}
+  namespace: {{ include "kagent.namespace" . }}
+  labels:
+    {{- include "kagent.labels" . | nindent 4 }}
+    app.kubernetes.io/component: database
+type: Opaque
+stringData:
+  {{ get $connectionStringSecretRef "key" | default "connectionString" }}: {{ $connectionString | quote }}
+{{- end }}