From fc8def2f478ef7013f80aef4452831c2263b22f3 Mon Sep 17 00:00:00 2001 From: Adam Moussa <166072409+amoussa1229@users.noreply.github.com> Date: Sat, 16 May 2026 22:10:34 -0400 Subject: [PATCH] Add Bedrock Knowledge Base, OpenSearch Serverless, and SQS message filtering - Provision OpenSearch Serverless collection for vector search - Create Bedrock Knowledge Base with Titan embedding model - Configure S3 data source with fixed-size chunking (512 tokens, 20% overlap) - Add suggestions Lambda with SQS event source filtering - Scope bedrock:InvokeModel IAM to specific model ARN patterns - Add internal API key secret in Secrets Manager - Add log retention (2 months) to all Lambda functions - Add docker-compose.yml for local PostgreSQL --- docker-compose.yml | 14 +++ infra/lib/compute-stack.ts | 186 +++++++++++++++++++++++++++++++++++-- 2 files changed, 194 insertions(+), 6 deletions(-) create mode 100644 docker-compose.yml diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..5031858 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,14 @@ +services: + postgres: + image: postgres:16-alpine + ports: + - "5432:5432" + environment: + POSTGRES_DB: proposalsystem + POSTGRES_USER: postgres + POSTGRES_PASSWORD: localdev + volumes: + - pgdata:/var/lib/postgresql/data + +volumes: + pgdata: diff --git a/infra/lib/compute-stack.ts b/infra/lib/compute-stack.ts index c5e0553..9ff5d48 100644 --- a/infra/lib/compute-stack.ts +++ b/infra/lib/compute-stack.ts @@ -9,6 +9,9 @@ import * as sqs from 'aws-cdk-lib/aws-sqs'; import * as cognito from 'aws-cdk-lib/aws-cognito'; import * as secretsmanager from 'aws-cdk-lib/aws-secretsmanager'; import * as lambdaEventSources from 'aws-cdk-lib/aws-lambda-event-sources'; +import * as bedrock from 'aws-cdk-lib/aws-bedrock'; +import * as opensearchserverless from 'aws-cdk-lib/aws-opensearchserverless'; +import * as logs from 'aws-cdk-lib/aws-logs'; import { Construct } from 'constructs'; export interface ComputeStackProps extends cdk.StackProps { @@ -28,6 +31,123 @@ export class ComputeStack extends cdk.Stack { const privateSubnets = { subnetType: ec2.SubnetType.PRIVATE_WITH_EGRESS }; + // Internal API key for Lambda-to-API calls (stored in Secrets Manager) + const internalApiKeySecret = new secretsmanager.Secret(this, 'InternalApiKeySecret', { + secretName: 'proposal-system/internal-api-key', + generateSecretString: { + excludePunctuation: true, + passwordLength: 48, + }, + }); + + // OpenSearch Serverless collection for Bedrock KB vector store + const ossEncryptionPolicy = new opensearchserverless.CfnSecurityPolicy(this, 'OssEncryptionPolicy', { + name: 'proposal-system-kb-enc', + type: 'encryption', + policy: JSON.stringify({ + Rules: [{ ResourceType: 'collection', Resource: ['collection/proposal-system-kb'] }], + AWSOwnedKey: true, + }), + }); + + const ossNetworkPolicy = new opensearchserverless.CfnSecurityPolicy(this, 'OssNetworkPolicy', { + name: 'proposal-system-kb-net', + type: 'network', + policy: JSON.stringify([{ + Rules: [ + { ResourceType: 'collection', Resource: ['collection/proposal-system-kb'] }, + { ResourceType: 'dashboard', Resource: ['collection/proposal-system-kb'] }, + ], + AllowFromPublic: true, + }]), + }); + + const ossCollection = new opensearchserverless.CfnCollection(this, 'OssCollection', { + name: 'proposal-system-kb', + type: 'VECTORSEARCH', + }); + ossCollection.addDependency(ossEncryptionPolicy); + ossCollection.addDependency(ossNetworkPolicy); + + // Bedrock KB execution role + const kbRole = new iam.Role(this, 'KnowledgeBaseRole', { + roleName: 'proposal-system-kb-role', + assumedBy: new iam.ServicePrincipal('bedrock.amazonaws.com'), + }); + + kbRole.addToPolicy(new iam.PolicyStatement({ + actions: ['s3:GetObject', 's3:ListBucket'], + resources: [props.libraryBucket.bucketArn, `${props.libraryBucket.bucketArn}/*`], + })); + + kbRole.addToPolicy(new iam.PolicyStatement({ + actions: ['aoss:APIAccessAll'], + resources: [ossCollection.attrArn], + })); + + kbRole.addToPolicy(new iam.PolicyStatement({ + actions: ['bedrock:InvokeModel'], + resources: [`arn:aws:bedrock:us-east-1::foundation-model/amazon.titan-embed-text-v2:0`], + })); + + // OpenSearch Serverless data access policy + new opensearchserverless.CfnAccessPolicy(this, 'OssDataAccessPolicy', { + name: 'proposal-system-kb-access', + type: 'data', + policy: JSON.stringify([{ + Rules: [ + { ResourceType: 'collection', Resource: ['collection/proposal-system-kb'], Permission: ['aoss:*'] }, + { ResourceType: 'index', Resource: ['index/proposal-system-kb/*'], Permission: ['aoss:*'] }, + ], + Principal: [kbRole.roleArn, `arn:aws:iam::${this.account}:root`], + }]), + }); + + // Bedrock Knowledge Base + const knowledgeBase = new bedrock.CfnKnowledgeBase(this, 'KnowledgeBase', { + name: 'proposal-system-kb', + roleArn: kbRole.roleArn, + knowledgeBaseConfiguration: { + type: 'VECTOR', + vectorKnowledgeBaseConfiguration: { + embeddingModelArn: `arn:aws:bedrock:us-east-1::foundation-model/amazon.titan-embed-text-v2:0`, + }, + }, + storageConfiguration: { + type: 'OPENSEARCH_SERVERLESS', + opensearchServerlessConfiguration: { + collectionArn: ossCollection.attrArn, + vectorIndexName: 'proposal-system-index', + fieldMapping: { + vectorField: 'embedding', + textField: 'text', + metadataField: 'metadata', + }, + }, + }, + }); + + // KB Data Source (S3 library bucket) + const dataSource = new bedrock.CfnDataSource(this, 'KbDataSource', { + name: 'proposal-system-library', + knowledgeBaseId: knowledgeBase.attrKnowledgeBaseId, + dataSourceConfiguration: { + type: 'S3', + s3Configuration: { + bucketArn: props.libraryBucket.bucketArn, + }, + }, + vectorIngestionConfiguration: { + chunkingConfiguration: { + chunkingStrategy: 'FIXED_SIZE', + fixedSizeChunkingConfiguration: { + maxTokens: 512, + overlapPercentage: 20, + }, + }, + }, + }); + // .NET 8 API Lambda const apiFunction = new lambda.Function(this, 'ApiFunction', { functionName: 'proposal-system-api', @@ -47,12 +167,15 @@ export class ComputeStack extends cdk.Stack { GENERATED_BUCKET: props.generatedBucket.bucketName, LIBRARY_BUCKET: props.libraryBucket.bucketName, JOBS_QUEUE_URL: props.jobsQueue.queueUrl, + INTERNAL_API_KEY_SECRET_ARN: internalApiKeySecret.secretArn, }, tracing: lambda.Tracing.ACTIVE, + logRetention: logs.RetentionDays.TWO_MONTHS, }); // API Lambda permissions props.dbSecret.grantRead(apiFunction); + internalApiKeySecret.grantRead(apiFunction); props.uploadsBucket.grantReadWrite(apiFunction); props.generatedBucket.grantRead(apiFunction); props.jobsQueue.grantSendMessages(apiFunction); @@ -93,6 +216,37 @@ export class ComputeStack extends cdk.Stack { integration: apiIntegration, }); + // Python Lambda: Suggestions Engine + const suggestionsFunction = new lambda.Function(this, 'SuggestionsFunction', { + functionName: 'proposal-system-suggestions', + runtime: lambda.Runtime.PYTHON_3_12, + architecture: lambda.Architecture.ARM_64, + handler: 'app.handler', + code: lambda.Code.fromAsset('../lambdas/suggestions'), + memorySize: 512, + timeout: cdk.Duration.seconds(60), + vpc: props.vpc, + vpcSubnets: privateSubnets, + securityGroups: [props.lambdaSecurityGroup], + environment: { + KNOWLEDGE_BASE_ID: knowledgeBase.attrKnowledgeBaseId, + MODEL_ID: 'us.anthropic.claude-sonnet-4-5-20250929-v1:0', + API_BASE_URL: httpApi.apiEndpoint, + INTERNAL_API_KEY_SECRET_ARN: internalApiKeySecret.secretArn, + }, + logRetention: logs.RetentionDays.TWO_MONTHS, + }); + + internalApiKeySecret.grantRead(suggestionsFunction); + suggestionsFunction.addToRolePolicy(new iam.PolicyStatement({ + actions: ['bedrock:InvokeModel'], + resources: [`arn:aws:bedrock:us-east-1::foundation-model/anthropic.claude-*`], + })); + suggestionsFunction.addToRolePolicy(new iam.PolicyStatement({ + actions: ['bedrock:Retrieve'], + resources: [knowledgeBase.attrKnowledgeBaseArn], + })); + // Python Lambda: PDF Extract const pdfExtractFunction = new lambda.Function(this, 'PdfExtractFunction', { functionName: 'proposal-system-pdf-extract', @@ -107,14 +261,18 @@ export class ComputeStack extends cdk.Stack { securityGroups: [props.lambdaSecurityGroup], environment: { UPLOADS_BUCKET: props.uploadsBucket.bucketName, + MODEL_ID: 'us.anthropic.claude-sonnet-4-5-20250929-v1:0', API_BASE_URL: httpApi.apiEndpoint, + INTERNAL_API_KEY_SECRET_ARN: internalApiKeySecret.secretArn, }, + logRetention: logs.RetentionDays.TWO_MONTHS, }); + internalApiKeySecret.grantRead(pdfExtractFunction); props.uploadsBucket.grantRead(pdfExtractFunction); pdfExtractFunction.addToRolePolicy(new iam.PolicyStatement({ actions: ['bedrock:InvokeModel'], - resources: ['*'], + resources: [`arn:aws:bedrock:us-east-1::foundation-model/anthropic.claude-*`], })); // Python Lambda: PDF Generate @@ -132,9 +290,12 @@ export class ComputeStack extends cdk.Stack { environment: { GENERATED_BUCKET: props.generatedBucket.bucketName, API_BASE_URL: httpApi.apiEndpoint, + INTERNAL_API_KEY_SECRET_ARN: internalApiKeySecret.secretArn, }, + logRetention: logs.RetentionDays.TWO_MONTHS, }); + internalApiKeySecret.grantRead(pdfGenerateFunction); props.generatedBucket.grantWrite(pdfGenerateFunction); // Python Lambda: Library Ingest @@ -151,20 +312,31 @@ export class ComputeStack extends cdk.Stack { securityGroups: [props.lambdaSecurityGroup], environment: { LIBRARY_BUCKET: props.libraryBucket.bucketName, + KNOWLEDGE_BASE_ID: knowledgeBase.attrKnowledgeBaseId, + DATA_SOURCE_ID: dataSource.attrDataSourceId, API_BASE_URL: httpApi.apiEndpoint, + INTERNAL_API_KEY_SECRET_ARN: internalApiKeySecret.secretArn, }, + logRetention: logs.RetentionDays.TWO_MONTHS, }); + internalApiKeySecret.grantRead(libraryIngestFunction); props.libraryBucket.grantWrite(libraryIngestFunction); libraryIngestFunction.addToRolePolicy(new iam.PolicyStatement({ actions: ['bedrock:StartIngestionJob'], - resources: ['*'], + resources: [knowledgeBase.attrKnowledgeBaseArn], + })); + + // SQS Event Sources with message filtering + suggestionsFunction.addEventSource(new lambdaEventSources.SqsEventSource(props.jobsQueue, { + batchSize: 1, + filters: [ + lambda.FilterCriteria.filter({ + body: { jobType: lambda.FilterRule.isEqual('suggestions') }, + }), + ], })); - // SQS Event Source for Python Lambdas - // All three consume from the same queue, routed by message attributes - // For now, use a single consumer that routes internally - // TODO: Phase 4 will refine this to use message filtering or separate queues per function pdfExtractFunction.addEventSource(new lambdaEventSources.SqsEventSource(props.jobsQueue, { batchSize: 1, filters: [ @@ -195,5 +367,7 @@ export class ComputeStack extends cdk.Stack { // Outputs new cdk.CfnOutput(this, 'ApiEndpoint', { value: httpApi.apiEndpoint }); new cdk.CfnOutput(this, 'ApiFunctionArn', { value: apiFunction.functionArn }); + new cdk.CfnOutput(this, 'KnowledgeBaseId', { value: knowledgeBase.attrKnowledgeBaseId }); + new cdk.CfnOutput(this, 'DataSourceId', { value: dataSource.attrDataSourceId }); } }