mirror of
https://github.com/Sea-Haven-Industries/proposal-system.git
synced 2026-09-30 06:33:13 +00:00
Add Bedrock Knowledge Base, OpenSearch Serverless, and SQS message filtering
- Provision OpenSearch Serverless collection for vector search - Create Bedrock Knowledge Base with Titan embedding model - Configure S3 data source with fixed-size chunking (512 tokens, 20% overlap) - Add suggestions Lambda with SQS event source filtering - Scope bedrock:InvokeModel IAM to specific model ARN patterns - Add internal API key secret in Secrets Manager - Add log retention (2 months) to all Lambda functions - Add docker-compose.yml for local PostgreSQL
This commit is contained in:
Adam Moussa
parent
9d6612e337
commit
fc8def2f47
2 changed files with 194 additions and 6 deletions
14
docker-compose.yml
Normal file
14
docker-compose.yml
Normal file
|
|
@ -0,0 +1,14 @@
|
|||
services:
|
||||
postgres:
|
||||
image: postgres:16-alpine
|
||||
ports:
|
||||
- "5432:5432"
|
||||
environment:
|
||||
POSTGRES_DB: proposalsystem
|
||||
POSTGRES_USER: postgres
|
||||
POSTGRES_PASSWORD: localdev
|
||||
volumes:
|
||||
- pgdata:/var/lib/postgresql/data
|
||||
|
||||
volumes:
|
||||
pgdata:
|
||||
|
|
@ -9,6 +9,9 @@ import * as sqs from 'aws-cdk-lib/aws-sqs';
|
|||
import * as cognito from 'aws-cdk-lib/aws-cognito';
|
||||
import * as secretsmanager from 'aws-cdk-lib/aws-secretsmanager';
|
||||
import * as lambdaEventSources from 'aws-cdk-lib/aws-lambda-event-sources';
|
||||
import * as bedrock from 'aws-cdk-lib/aws-bedrock';
|
||||
import * as opensearchserverless from 'aws-cdk-lib/aws-opensearchserverless';
|
||||
import * as logs from 'aws-cdk-lib/aws-logs';
|
||||
import { Construct } from 'constructs';
|
||||
|
||||
export interface ComputeStackProps extends cdk.StackProps {
|
||||
|
|
@ -28,6 +31,123 @@ export class ComputeStack extends cdk.Stack {
|
|||
|
||||
const privateSubnets = { subnetType: ec2.SubnetType.PRIVATE_WITH_EGRESS };
|
||||
|
||||
// Internal API key for Lambda-to-API calls (stored in Secrets Manager)
|
||||
const internalApiKeySecret = new secretsmanager.Secret(this, 'InternalApiKeySecret', {
|
||||
secretName: 'proposal-system/internal-api-key',
|
||||
generateSecretString: {
|
||||
excludePunctuation: true,
|
||||
passwordLength: 48,
|
||||
},
|
||||
});
|
||||
|
||||
// OpenSearch Serverless collection for Bedrock KB vector store
|
||||
const ossEncryptionPolicy = new opensearchserverless.CfnSecurityPolicy(this, 'OssEncryptionPolicy', {
|
||||
name: 'proposal-system-kb-enc',
|
||||
type: 'encryption',
|
||||
policy: JSON.stringify({
|
||||
Rules: [{ ResourceType: 'collection', Resource: ['collection/proposal-system-kb'] }],
|
||||
AWSOwnedKey: true,
|
||||
}),
|
||||
});
|
||||
|
||||
const ossNetworkPolicy = new opensearchserverless.CfnSecurityPolicy(this, 'OssNetworkPolicy', {
|
||||
name: 'proposal-system-kb-net',
|
||||
type: 'network',
|
||||
policy: JSON.stringify([{
|
||||
Rules: [
|
||||
{ ResourceType: 'collection', Resource: ['collection/proposal-system-kb'] },
|
||||
{ ResourceType: 'dashboard', Resource: ['collection/proposal-system-kb'] },
|
||||
],
|
||||
AllowFromPublic: true,
|
||||
}]),
|
||||
});
|
||||
|
||||
const ossCollection = new opensearchserverless.CfnCollection(this, 'OssCollection', {
|
||||
name: 'proposal-system-kb',
|
||||
type: 'VECTORSEARCH',
|
||||
});
|
||||
ossCollection.addDependency(ossEncryptionPolicy);
|
||||
ossCollection.addDependency(ossNetworkPolicy);
|
||||
|
||||
// Bedrock KB execution role
|
||||
const kbRole = new iam.Role(this, 'KnowledgeBaseRole', {
|
||||
roleName: 'proposal-system-kb-role',
|
||||
assumedBy: new iam.ServicePrincipal('bedrock.amazonaws.com'),
|
||||
});
|
||||
|
||||
kbRole.addToPolicy(new iam.PolicyStatement({
|
||||
actions: ['s3:GetObject', 's3:ListBucket'],
|
||||
resources: [props.libraryBucket.bucketArn, `${props.libraryBucket.bucketArn}/*`],
|
||||
}));
|
||||
|
||||
kbRole.addToPolicy(new iam.PolicyStatement({
|
||||
actions: ['aoss:APIAccessAll'],
|
||||
resources: [ossCollection.attrArn],
|
||||
}));
|
||||
|
||||
kbRole.addToPolicy(new iam.PolicyStatement({
|
||||
actions: ['bedrock:InvokeModel'],
|
||||
resources: [`arn:aws:bedrock:us-east-1::foundation-model/amazon.titan-embed-text-v2:0`],
|
||||
}));
|
||||
|
||||
// OpenSearch Serverless data access policy
|
||||
new opensearchserverless.CfnAccessPolicy(this, 'OssDataAccessPolicy', {
|
||||
name: 'proposal-system-kb-access',
|
||||
type: 'data',
|
||||
policy: JSON.stringify([{
|
||||
Rules: [
|
||||
{ ResourceType: 'collection', Resource: ['collection/proposal-system-kb'], Permission: ['aoss:*'] },
|
||||
{ ResourceType: 'index', Resource: ['index/proposal-system-kb/*'], Permission: ['aoss:*'] },
|
||||
],
|
||||
Principal: [kbRole.roleArn, `arn:aws:iam::${this.account}:root`],
|
||||
}]),
|
||||
});
|
||||
|
||||
// Bedrock Knowledge Base
|
||||
const knowledgeBase = new bedrock.CfnKnowledgeBase(this, 'KnowledgeBase', {
|
||||
name: 'proposal-system-kb',
|
||||
roleArn: kbRole.roleArn,
|
||||
knowledgeBaseConfiguration: {
|
||||
type: 'VECTOR',
|
||||
vectorKnowledgeBaseConfiguration: {
|
||||
embeddingModelArn: `arn:aws:bedrock:us-east-1::foundation-model/amazon.titan-embed-text-v2:0`,
|
||||
},
|
||||
},
|
||||
storageConfiguration: {
|
||||
type: 'OPENSEARCH_SERVERLESS',
|
||||
opensearchServerlessConfiguration: {
|
||||
collectionArn: ossCollection.attrArn,
|
||||
vectorIndexName: 'proposal-system-index',
|
||||
fieldMapping: {
|
||||
vectorField: 'embedding',
|
||||
textField: 'text',
|
||||
metadataField: 'metadata',
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
// KB Data Source (S3 library bucket)
|
||||
const dataSource = new bedrock.CfnDataSource(this, 'KbDataSource', {
|
||||
name: 'proposal-system-library',
|
||||
knowledgeBaseId: knowledgeBase.attrKnowledgeBaseId,
|
||||
dataSourceConfiguration: {
|
||||
type: 'S3',
|
||||
s3Configuration: {
|
||||
bucketArn: props.libraryBucket.bucketArn,
|
||||
},
|
||||
},
|
||||
vectorIngestionConfiguration: {
|
||||
chunkingConfiguration: {
|
||||
chunkingStrategy: 'FIXED_SIZE',
|
||||
fixedSizeChunkingConfiguration: {
|
||||
maxTokens: 512,
|
||||
overlapPercentage: 20,
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
// .NET 8 API Lambda
|
||||
const apiFunction = new lambda.Function(this, 'ApiFunction', {
|
||||
functionName: 'proposal-system-api',
|
||||
|
|
@ -47,12 +167,15 @@ export class ComputeStack extends cdk.Stack {
|
|||
GENERATED_BUCKET: props.generatedBucket.bucketName,
|
||||
LIBRARY_BUCKET: props.libraryBucket.bucketName,
|
||||
JOBS_QUEUE_URL: props.jobsQueue.queueUrl,
|
||||
INTERNAL_API_KEY_SECRET_ARN: internalApiKeySecret.secretArn,
|
||||
},
|
||||
tracing: lambda.Tracing.ACTIVE,
|
||||
logRetention: logs.RetentionDays.TWO_MONTHS,
|
||||
});
|
||||
|
||||
// API Lambda permissions
|
||||
props.dbSecret.grantRead(apiFunction);
|
||||
internalApiKeySecret.grantRead(apiFunction);
|
||||
props.uploadsBucket.grantReadWrite(apiFunction);
|
||||
props.generatedBucket.grantRead(apiFunction);
|
||||
props.jobsQueue.grantSendMessages(apiFunction);
|
||||
|
|
@ -93,6 +216,37 @@ export class ComputeStack extends cdk.Stack {
|
|||
integration: apiIntegration,
|
||||
});
|
||||
|
||||
// Python Lambda: Suggestions Engine
|
||||
const suggestionsFunction = new lambda.Function(this, 'SuggestionsFunction', {
|
||||
functionName: 'proposal-system-suggestions',
|
||||
runtime: lambda.Runtime.PYTHON_3_12,
|
||||
architecture: lambda.Architecture.ARM_64,
|
||||
handler: 'app.handler',
|
||||
code: lambda.Code.fromAsset('../lambdas/suggestions'),
|
||||
memorySize: 512,
|
||||
timeout: cdk.Duration.seconds(60),
|
||||
vpc: props.vpc,
|
||||
vpcSubnets: privateSubnets,
|
||||
securityGroups: [props.lambdaSecurityGroup],
|
||||
environment: {
|
||||
KNOWLEDGE_BASE_ID: knowledgeBase.attrKnowledgeBaseId,
|
||||
MODEL_ID: 'us.anthropic.claude-sonnet-4-5-20250929-v1:0',
|
||||
API_BASE_URL: httpApi.apiEndpoint,
|
||||
INTERNAL_API_KEY_SECRET_ARN: internalApiKeySecret.secretArn,
|
||||
},
|
||||
logRetention: logs.RetentionDays.TWO_MONTHS,
|
||||
});
|
||||
|
||||
internalApiKeySecret.grantRead(suggestionsFunction);
|
||||
suggestionsFunction.addToRolePolicy(new iam.PolicyStatement({
|
||||
actions: ['bedrock:InvokeModel'],
|
||||
resources: [`arn:aws:bedrock:us-east-1::foundation-model/anthropic.claude-*`],
|
||||
}));
|
||||
suggestionsFunction.addToRolePolicy(new iam.PolicyStatement({
|
||||
actions: ['bedrock:Retrieve'],
|
||||
resources: [knowledgeBase.attrKnowledgeBaseArn],
|
||||
}));
|
||||
|
||||
// Python Lambda: PDF Extract
|
||||
const pdfExtractFunction = new lambda.Function(this, 'PdfExtractFunction', {
|
||||
functionName: 'proposal-system-pdf-extract',
|
||||
|
|
@ -107,14 +261,18 @@ export class ComputeStack extends cdk.Stack {
|
|||
securityGroups: [props.lambdaSecurityGroup],
|
||||
environment: {
|
||||
UPLOADS_BUCKET: props.uploadsBucket.bucketName,
|
||||
MODEL_ID: 'us.anthropic.claude-sonnet-4-5-20250929-v1:0',
|
||||
API_BASE_URL: httpApi.apiEndpoint,
|
||||
INTERNAL_API_KEY_SECRET_ARN: internalApiKeySecret.secretArn,
|
||||
},
|
||||
logRetention: logs.RetentionDays.TWO_MONTHS,
|
||||
});
|
||||
|
||||
internalApiKeySecret.grantRead(pdfExtractFunction);
|
||||
props.uploadsBucket.grantRead(pdfExtractFunction);
|
||||
pdfExtractFunction.addToRolePolicy(new iam.PolicyStatement({
|
||||
actions: ['bedrock:InvokeModel'],
|
||||
resources: ['*'],
|
||||
resources: [`arn:aws:bedrock:us-east-1::foundation-model/anthropic.claude-*`],
|
||||
}));
|
||||
|
||||
// Python Lambda: PDF Generate
|
||||
|
|
@ -132,9 +290,12 @@ export class ComputeStack extends cdk.Stack {
|
|||
environment: {
|
||||
GENERATED_BUCKET: props.generatedBucket.bucketName,
|
||||
API_BASE_URL: httpApi.apiEndpoint,
|
||||
INTERNAL_API_KEY_SECRET_ARN: internalApiKeySecret.secretArn,
|
||||
},
|
||||
logRetention: logs.RetentionDays.TWO_MONTHS,
|
||||
});
|
||||
|
||||
internalApiKeySecret.grantRead(pdfGenerateFunction);
|
||||
props.generatedBucket.grantWrite(pdfGenerateFunction);
|
||||
|
||||
// Python Lambda: Library Ingest
|
||||
|
|
@ -151,20 +312,31 @@ export class ComputeStack extends cdk.Stack {
|
|||
securityGroups: [props.lambdaSecurityGroup],
|
||||
environment: {
|
||||
LIBRARY_BUCKET: props.libraryBucket.bucketName,
|
||||
KNOWLEDGE_BASE_ID: knowledgeBase.attrKnowledgeBaseId,
|
||||
DATA_SOURCE_ID: dataSource.attrDataSourceId,
|
||||
API_BASE_URL: httpApi.apiEndpoint,
|
||||
INTERNAL_API_KEY_SECRET_ARN: internalApiKeySecret.secretArn,
|
||||
},
|
||||
logRetention: logs.RetentionDays.TWO_MONTHS,
|
||||
});
|
||||
|
||||
internalApiKeySecret.grantRead(libraryIngestFunction);
|
||||
props.libraryBucket.grantWrite(libraryIngestFunction);
|
||||
libraryIngestFunction.addToRolePolicy(new iam.PolicyStatement({
|
||||
actions: ['bedrock:StartIngestionJob'],
|
||||
resources: ['*'],
|
||||
resources: [knowledgeBase.attrKnowledgeBaseArn],
|
||||
}));
|
||||
|
||||
// SQS Event Sources with message filtering
|
||||
suggestionsFunction.addEventSource(new lambdaEventSources.SqsEventSource(props.jobsQueue, {
|
||||
batchSize: 1,
|
||||
filters: [
|
||||
lambda.FilterCriteria.filter({
|
||||
body: { jobType: lambda.FilterRule.isEqual('suggestions') },
|
||||
}),
|
||||
],
|
||||
}));
|
||||
|
||||
// SQS Event Source for Python Lambdas
|
||||
// All three consume from the same queue, routed by message attributes
|
||||
// For now, use a single consumer that routes internally
|
||||
// TODO: Phase 4 will refine this to use message filtering or separate queues per function
|
||||
pdfExtractFunction.addEventSource(new lambdaEventSources.SqsEventSource(props.jobsQueue, {
|
||||
batchSize: 1,
|
||||
filters: [
|
||||
|
|
@ -195,5 +367,7 @@ export class ComputeStack extends cdk.Stack {
|
|||
// Outputs
|
||||
new cdk.CfnOutput(this, 'ApiEndpoint', { value: httpApi.apiEndpoint });
|
||||
new cdk.CfnOutput(this, 'ApiFunctionArn', { value: apiFunction.functionArn });
|
||||
new cdk.CfnOutput(this, 'KnowledgeBaseId', { value: knowledgeBase.attrKnowledgeBaseId });
|
||||
new cdk.CfnOutput(this, 'DataSourceId', { value: dataSource.attrDataSourceId });
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue