Initial commit: Telegram Management System
Some checks failed
Deploy / deploy (push) Has been cancelled
Some checks failed
Deploy / deploy (push) Has been cancelled
Full-stack web application for Telegram management - Frontend: Vue 3 + Vben Admin - Backend: NestJS - Features: User management, group broadcast, statistics 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
你的用户名
1
marketing-agent/services/safety-guard/src/app.js
Normal file
1
marketing-agent/services/safety-guard/src/app.js
Normal file
@@ -0,0 +1 @@
|
||||
import './index.js';
|
||||
49
marketing-agent/services/safety-guard/src/config/database.js
Normal file
49
marketing-agent/services/safety-guard/src/config/database.js
Normal file
@@ -0,0 +1,49 @@
|
||||
import mongoose from 'mongoose';
|
||||
import { logger } from '../utils/logger.js';
|
||||
|
||||
export const connectDatabase = async () => {
|
||||
const mongoUri = process.env.MONGODB_URI || 'mongodb://localhost:27017/safety-guard';
|
||||
|
||||
const options = {
|
||||
useNewUrlParser: true,
|
||||
useUnifiedTopology: true,
|
||||
autoIndex: true,
|
||||
maxPoolSize: 10,
|
||||
serverSelectionTimeoutMS: 5000,
|
||||
socketTimeoutMS: 45000,
|
||||
family: 4
|
||||
};
|
||||
|
||||
try {
|
||||
await mongoose.connect(mongoUri, options);
|
||||
logger.info('MongoDB connected successfully');
|
||||
|
||||
// Handle connection events
|
||||
mongoose.connection.on('error', (err) => {
|
||||
logger.error('MongoDB connection error:', err);
|
||||
});
|
||||
|
||||
mongoose.connection.on('disconnected', () => {
|
||||
logger.warn('MongoDB disconnected');
|
||||
});
|
||||
|
||||
mongoose.connection.on('reconnected', () => {
|
||||
logger.info('MongoDB reconnected');
|
||||
});
|
||||
|
||||
return mongoose.connection;
|
||||
} catch (error) {
|
||||
logger.error('Failed to connect to MongoDB:', error);
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
|
||||
export const disconnectDatabase = async () => {
|
||||
try {
|
||||
await mongoose.disconnect();
|
||||
logger.info('MongoDB disconnected successfully');
|
||||
} catch (error) {
|
||||
logger.error('Error disconnecting from MongoDB:', error);
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
156
marketing-agent/services/safety-guard/src/config/redis.js
Normal file
156
marketing-agent/services/safety-guard/src/config/redis.js
Normal file
@@ -0,0 +1,156 @@
|
||||
import Redis from 'ioredis';
|
||||
import { logger } from '../utils/logger.js';
|
||||
|
||||
export class RedisClient {
|
||||
constructor() {
|
||||
this.client = null;
|
||||
}
|
||||
|
||||
static getInstance() {
|
||||
if (!RedisClient.instance) {
|
||||
RedisClient.instance = new RedisClient();
|
||||
}
|
||||
return RedisClient.instance;
|
||||
}
|
||||
|
||||
async connect() {
|
||||
const config = {
|
||||
host: process.env.REDIS_HOST || 'localhost',
|
||||
port: process.env.REDIS_PORT || 6379,
|
||||
password: process.env.REDIS_PASSWORD || undefined,
|
||||
db: parseInt(process.env.REDIS_DB) || 2, // Different DB for safety guard
|
||||
retryStrategy: (times) => {
|
||||
const delay = Math.min(times * 50, 2000);
|
||||
return delay;
|
||||
},
|
||||
enableOfflineQueue: false
|
||||
};
|
||||
|
||||
try {
|
||||
this.client = new Redis(config);
|
||||
|
||||
this.client.on('connect', () => {
|
||||
logger.info('Redis connection established');
|
||||
});
|
||||
|
||||
this.client.on('error', (err) => {
|
||||
logger.error('Redis error:', err);
|
||||
});
|
||||
|
||||
this.client.on('close', () => {
|
||||
logger.warn('Redis connection closed');
|
||||
});
|
||||
|
||||
// Wait for connection
|
||||
await this.client.ping();
|
||||
|
||||
return this.client;
|
||||
} catch (error) {
|
||||
logger.error('Failed to connect to Redis:', error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async checkHealth() {
|
||||
try {
|
||||
const result = await this.client.ping();
|
||||
return result === 'PONG';
|
||||
} catch (error) {
|
||||
logger.error('Redis health check failed:', error);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
async disconnect() {
|
||||
if (this.client) {
|
||||
await this.client.quit();
|
||||
logger.info('Redis connection closed');
|
||||
}
|
||||
}
|
||||
|
||||
// Cache methods with JSON serialization
|
||||
async setWithExpiry(key, value, ttl) {
|
||||
return await this.client.setex(key, ttl, JSON.stringify(value));
|
||||
}
|
||||
|
||||
async get(key) {
|
||||
const value = await this.client.get(key);
|
||||
return value ? JSON.parse(value) : null;
|
||||
}
|
||||
|
||||
async del(key) {
|
||||
return await this.client.del(key);
|
||||
}
|
||||
|
||||
async exists(key) {
|
||||
return await this.client.exists(key);
|
||||
}
|
||||
|
||||
// Hash operations
|
||||
async hset(key, field, value) {
|
||||
return await this.client.hset(key, field, JSON.stringify(value));
|
||||
}
|
||||
|
||||
async hget(key, field) {
|
||||
const value = await this.client.hget(key, field);
|
||||
return value ? JSON.parse(value) : null;
|
||||
}
|
||||
|
||||
async hdel(key, field) {
|
||||
return await this.client.hdel(key, field);
|
||||
}
|
||||
|
||||
async hgetall(key) {
|
||||
const data = await this.client.hgetall(key);
|
||||
const result = {};
|
||||
for (const [field, value] of Object.entries(data)) {
|
||||
result[field] = JSON.parse(value);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
// List operations
|
||||
async lpush(key, value) {
|
||||
return await this.client.lpush(key, JSON.stringify(value));
|
||||
}
|
||||
|
||||
async rpush(key, value) {
|
||||
return await this.client.rpush(key, JSON.stringify(value));
|
||||
}
|
||||
|
||||
async lrange(key, start, stop) {
|
||||
const items = await this.client.lrange(key, start, stop);
|
||||
return items.map(item => JSON.parse(item));
|
||||
}
|
||||
|
||||
async ltrim(key, start, stop) {
|
||||
return await this.client.ltrim(key, start, stop);
|
||||
}
|
||||
|
||||
// Set operations
|
||||
async sadd(key, member) {
|
||||
return await this.client.sadd(key, JSON.stringify(member));
|
||||
}
|
||||
|
||||
async srem(key, member) {
|
||||
return await this.client.srem(key, JSON.stringify(member));
|
||||
}
|
||||
|
||||
async smembers(key) {
|
||||
const members = await this.client.smembers(key);
|
||||
return members.map(member => JSON.parse(member));
|
||||
}
|
||||
|
||||
async sismember(key, member) {
|
||||
return await this.client.sismember(key, JSON.stringify(member));
|
||||
}
|
||||
|
||||
// Expiry operations
|
||||
async expire(key, seconds) {
|
||||
return await this.client.expire(key, seconds);
|
||||
}
|
||||
|
||||
async ttl(key) {
|
||||
return await this.client.ttl(key);
|
||||
}
|
||||
}
|
||||
105
marketing-agent/services/safety-guard/src/index.js
Normal file
105
marketing-agent/services/safety-guard/src/index.js
Normal file
@@ -0,0 +1,105 @@
|
||||
import 'dotenv/config';
|
||||
import Hapi from '@hapi/hapi';
|
||||
import { logger } from './utils/logger.js';
|
||||
import { connectDatabase } from './config/database.js';
|
||||
import { RedisClient } from './config/redis.js';
|
||||
import routes from './routes/index.js';
|
||||
import { ModerationService } from './services/ModerationService.js';
|
||||
import { ComplianceService } from './services/ComplianceService.js';
|
||||
import { RateLimiterService } from './services/RateLimiterService.js';
|
||||
import { ContentPolicyService } from './services/ContentPolicyService.js';
|
||||
|
||||
const init = async () => {
|
||||
// Initialize database connections
|
||||
await connectDatabase();
|
||||
const redisClient = RedisClient.getInstance();
|
||||
await redisClient.connect();
|
||||
|
||||
// Initialize services
|
||||
ModerationService.getInstance();
|
||||
ComplianceService.getInstance();
|
||||
RateLimiterService.getInstance();
|
||||
ContentPolicyService.getInstance();
|
||||
|
||||
// Create Hapi server
|
||||
const server = Hapi.server({
|
||||
port: process.env.PORT || 3004,
|
||||
host: process.env.HOST || 'localhost',
|
||||
routes: {
|
||||
cors: {
|
||||
origin: ['*'],
|
||||
headers: ['Accept', 'Content-Type', 'Authorization'],
|
||||
credentials: true
|
||||
},
|
||||
payload: {
|
||||
maxBytes: 10485760 // 10MB
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// Register routes
|
||||
server.route(routes);
|
||||
|
||||
// Health check endpoint
|
||||
server.route({
|
||||
method: 'GET',
|
||||
path: '/health',
|
||||
handler: async (request, h) => {
|
||||
const dbHealth = await checkDatabaseHealth();
|
||||
const redisHealth = await redisClient.checkHealth();
|
||||
|
||||
const isHealthy = dbHealth && redisHealth;
|
||||
|
||||
return h.response({
|
||||
status: isHealthy ? 'healthy' : 'unhealthy',
|
||||
service: 'safety-guard',
|
||||
timestamp: new Date().toISOString(),
|
||||
checks: {
|
||||
database: dbHealth,
|
||||
redis: redisHealth
|
||||
}
|
||||
}).code(isHealthy ? 200 : 503);
|
||||
}
|
||||
});
|
||||
|
||||
// Start server
|
||||
await server.start();
|
||||
logger.info(`Safety Guard service running on ${server.info.uri}`);
|
||||
|
||||
// Graceful shutdown
|
||||
process.on('SIGTERM', async () => {
|
||||
logger.info('SIGTERM signal received');
|
||||
await server.stop({ timeout: 10000 });
|
||||
await redisClient.disconnect();
|
||||
process.exit(0);
|
||||
});
|
||||
|
||||
process.on('SIGINT', async () => {
|
||||
logger.info('SIGINT signal received');
|
||||
await server.stop({ timeout: 10000 });
|
||||
await redisClient.disconnect();
|
||||
process.exit(0);
|
||||
});
|
||||
};
|
||||
|
||||
const checkDatabaseHealth = async () => {
|
||||
try {
|
||||
const { connection } = await import('mongoose');
|
||||
return connection.readyState === 1;
|
||||
} catch (error) {
|
||||
logger.error('Database health check failed:', error);
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
// Handle unhandled rejections
|
||||
process.on('unhandledRejection', (err) => {
|
||||
logger.error('Unhandled rejection:', err);
|
||||
process.exit(1);
|
||||
});
|
||||
|
||||
// Start the service
|
||||
init().catch((err) => {
|
||||
logger.error('Failed to start Safety Guard service:', err);
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -0,0 +1,62 @@
|
||||
import mongoose from 'mongoose';
|
||||
|
||||
const complianceCheckSchema = new mongoose.Schema({
|
||||
// Multi-tenant support
|
||||
tenantId: {
|
||||
type: mongoose.Schema.Types.ObjectId,
|
||||
ref: 'Tenant',
|
||||
required: true,
|
||||
index: true
|
||||
},
|
||||
action: {
|
||||
type: String,
|
||||
required: true
|
||||
},
|
||||
targetRegion: {
|
||||
type: String,
|
||||
required: true
|
||||
},
|
||||
violations: [{
|
||||
type: String,
|
||||
severity: {
|
||||
type: String,
|
||||
enum: ['low', 'medium', 'high'],
|
||||
required: true
|
||||
},
|
||||
message: String,
|
||||
regulation: String
|
||||
}],
|
||||
warnings: [{
|
||||
type: String,
|
||||
severity: {
|
||||
type: String,
|
||||
enum: ['low', 'medium', 'high'],
|
||||
required: true
|
||||
},
|
||||
message: String,
|
||||
category: String
|
||||
}],
|
||||
processingTime: {
|
||||
type: Number,
|
||||
required: true
|
||||
},
|
||||
timestamp: {
|
||||
type: Date,
|
||||
default: Date.now,
|
||||
index: true
|
||||
}
|
||||
}, {
|
||||
timestamps: true
|
||||
});
|
||||
|
||||
// Indexes
|
||||
complianceCheckSchema.index({ timestamp: -1 });
|
||||
complianceCheckSchema.index({ targetRegion: 1, action: 1 });
|
||||
complianceCheckSchema.index({ 'violations.type': 1 });
|
||||
|
||||
// Multi-tenant indexes
|
||||
complianceCheckSchema.index({ tenantId: 1, timestamp: -1 });
|
||||
complianceCheckSchema.index({ tenantId: 1, targetRegion: 1, action: 1 });
|
||||
complianceCheckSchema.index({ tenantId: 1, 'violations.type': 1 });
|
||||
|
||||
export const ComplianceCheck = mongoose.model('ComplianceCheck', complianceCheckSchema);
|
||||
@@ -0,0 +1,73 @@
|
||||
import mongoose from 'mongoose';
|
||||
|
||||
const complianceRuleSchema = new mongoose.Schema({
|
||||
// Multi-tenant support
|
||||
tenantId: {
|
||||
type: mongoose.Schema.Types.ObjectId,
|
||||
ref: 'Tenant',
|
||||
required: true,
|
||||
index: true
|
||||
},
|
||||
ruleId: {
|
||||
type: String,
|
||||
required: true,
|
||||
unique: true
|
||||
},
|
||||
name: {
|
||||
type: String,
|
||||
required: true
|
||||
},
|
||||
description: String,
|
||||
regulation: {
|
||||
type: String,
|
||||
enum: ['GDPR', 'CCPA', 'LGPD', 'POPIA', 'COPPA', 'CAN-SPAM', 'PECR', 'PIPEDA', 'CUSTOM'],
|
||||
required: true
|
||||
},
|
||||
region: {
|
||||
type: String,
|
||||
required: true
|
||||
},
|
||||
requirements: {
|
||||
consentRequired: Boolean,
|
||||
doubleOptIn: Boolean,
|
||||
rightToDelete: Boolean,
|
||||
dataPortability: Boolean,
|
||||
dataRetentionDays: Number,
|
||||
minAge: Number,
|
||||
explicitOptOut: Boolean,
|
||||
crossBorderRestrictions: Boolean
|
||||
},
|
||||
validations: [{
|
||||
field: String,
|
||||
operator: {
|
||||
type: String,
|
||||
enum: ['equals', 'contains', 'regex', 'min', 'max', 'exists']
|
||||
},
|
||||
value: mongoose.Schema.Types.Mixed,
|
||||
severity: {
|
||||
type: String,
|
||||
enum: ['low', 'medium', 'high'],
|
||||
default: 'medium'
|
||||
}
|
||||
}],
|
||||
active: {
|
||||
type: Boolean,
|
||||
default: true
|
||||
},
|
||||
effectiveDate: Date,
|
||||
expiryDate: Date
|
||||
}, {
|
||||
timestamps: true
|
||||
});
|
||||
|
||||
// Indexes
|
||||
complianceRuleSchema.index({ ruleId: 1 });
|
||||
complianceRuleSchema.index({ region: 1, active: 1 });
|
||||
complianceRuleSchema.index({ regulation: 1 });
|
||||
|
||||
// Multi-tenant indexes
|
||||
complianceRuleSchema.index({ tenantId: 1, ruleId: 1 });
|
||||
complianceRuleSchema.index({ tenantId: 1, region: 1, active: 1 });
|
||||
complianceRuleSchema.index({ tenantId: 1, regulation: 1 });
|
||||
|
||||
export const ComplianceRule = mongoose.model('ComplianceRule', complianceRuleSchema);
|
||||
@@ -0,0 +1,82 @@
|
||||
import mongoose from 'mongoose';
|
||||
|
||||
const contentPolicySchema = new mongoose.Schema({
|
||||
// Multi-tenant support
|
||||
tenantId: {
|
||||
type: mongoose.Schema.Types.ObjectId,
|
||||
ref: 'Tenant',
|
||||
required: true,
|
||||
index: true
|
||||
},
|
||||
policyId: {
|
||||
type: String,
|
||||
required: true,
|
||||
unique: true
|
||||
},
|
||||
name: {
|
||||
type: String,
|
||||
required: true
|
||||
},
|
||||
description: String,
|
||||
rules: {
|
||||
maxMessageLength: {
|
||||
type: Number,
|
||||
default: 4096
|
||||
},
|
||||
minMessageLength: {
|
||||
type: Number,
|
||||
default: 1
|
||||
},
|
||||
allowedContentTypes: [{
|
||||
type: String,
|
||||
enum: ['text', 'image', 'video', 'audio', 'document', 'location', 'contact']
|
||||
}],
|
||||
prohibitedPatterns: [String],
|
||||
maxMediaSize: Number,
|
||||
maxRiskScore: {
|
||||
type: Number,
|
||||
min: 0,
|
||||
max: 1,
|
||||
default: 0.7
|
||||
},
|
||||
requireModeration: {
|
||||
type: Boolean,
|
||||
default: true
|
||||
},
|
||||
requireCompliance: {
|
||||
type: Boolean,
|
||||
default: true
|
||||
},
|
||||
requireOptOut: Boolean,
|
||||
maxLinksPerMessage: Number,
|
||||
requireCTA: Boolean,
|
||||
additionalChecks: [{
|
||||
type: String,
|
||||
enum: ['duplicate_detection', 'spam_detection', 'link_validation', 'language_detection']
|
||||
}]
|
||||
},
|
||||
active: {
|
||||
type: Boolean,
|
||||
default: true
|
||||
},
|
||||
priority: {
|
||||
type: Number,
|
||||
default: 0
|
||||
},
|
||||
applicableRegions: [String],
|
||||
applicableActions: [String]
|
||||
}, {
|
||||
timestamps: true
|
||||
});
|
||||
|
||||
// Indexes
|
||||
contentPolicySchema.index({ policyId: 1 });
|
||||
contentPolicySchema.index({ active: 1, priority: -1 });
|
||||
contentPolicySchema.index({ applicableRegions: 1 });
|
||||
|
||||
// Multi-tenant indexes
|
||||
contentPolicySchema.index({ tenantId: 1, policyId: 1 });
|
||||
contentPolicySchema.index({ tenantId: 1, active: 1, priority: -1 });
|
||||
contentPolicySchema.index({ tenantId: 1, applicableRegions: 1 });
|
||||
|
||||
export const ContentPolicy = mongoose.model('ContentPolicy', contentPolicySchema);
|
||||
@@ -0,0 +1,60 @@
|
||||
import mongoose from 'mongoose';
|
||||
|
||||
const contentViolationSchema = new mongoose.Schema({
|
||||
// Multi-tenant support
|
||||
tenantId: {
|
||||
type: mongoose.Schema.Types.ObjectId,
|
||||
ref: 'Tenant',
|
||||
required: true,
|
||||
index: true
|
||||
},
|
||||
content: {
|
||||
type: String,
|
||||
required: true,
|
||||
maxlength: 1000 // Store only preview
|
||||
},
|
||||
violations: [{
|
||||
type: {
|
||||
type: String,
|
||||
required: true
|
||||
},
|
||||
severity: {
|
||||
type: String,
|
||||
enum: ['low', 'medium', 'high', 'critical'],
|
||||
required: true
|
||||
},
|
||||
details: mongoose.Schema.Types.Mixed
|
||||
}],
|
||||
riskScore: {
|
||||
type: Number,
|
||||
required: true,
|
||||
min: 0,
|
||||
max: 1
|
||||
},
|
||||
sentiment: {
|
||||
score: Number,
|
||||
comparative: Number,
|
||||
positive: Number,
|
||||
negative: Number,
|
||||
neutral: Number
|
||||
},
|
||||
timestamp: {
|
||||
type: Date,
|
||||
default: Date.now,
|
||||
index: true
|
||||
}
|
||||
}, {
|
||||
timestamps: true
|
||||
});
|
||||
|
||||
// Indexes
|
||||
contentViolationSchema.index({ timestamp: -1 });
|
||||
contentViolationSchema.index({ 'violations.type': 1 });
|
||||
contentViolationSchema.index({ riskScore: -1 });
|
||||
|
||||
// Multi-tenant indexes
|
||||
contentViolationSchema.index({ tenantId: 1, timestamp: -1 });
|
||||
contentViolationSchema.index({ tenantId: 1, 'violations.type': 1 });
|
||||
contentViolationSchema.index({ tenantId: 1, riskScore: -1 });
|
||||
|
||||
export const ContentViolation = mongoose.model('ContentViolation', contentViolationSchema);
|
||||
@@ -0,0 +1,64 @@
|
||||
import mongoose from 'mongoose';
|
||||
|
||||
const policyViolationSchema = new mongoose.Schema({
|
||||
// Multi-tenant support
|
||||
tenantId: {
|
||||
type: mongoose.Schema.Types.ObjectId,
|
||||
ref: 'Tenant',
|
||||
required: true,
|
||||
index: true
|
||||
},
|
||||
policyId: {
|
||||
type: String,
|
||||
required: true
|
||||
},
|
||||
contentPreview: {
|
||||
type: String,
|
||||
maxlength: 200
|
||||
},
|
||||
violations: [{
|
||||
type: String,
|
||||
severity: {
|
||||
type: String,
|
||||
enum: ['low', 'medium', 'high', 'critical'],
|
||||
required: true
|
||||
},
|
||||
message: String,
|
||||
details: mongoose.Schema.Types.Mixed
|
||||
}],
|
||||
warnings: [{
|
||||
type: String,
|
||||
severity: {
|
||||
type: String,
|
||||
enum: ['low', 'medium', 'high'],
|
||||
required: true
|
||||
},
|
||||
message: String
|
||||
}],
|
||||
moderationResult: mongoose.Schema.Types.Mixed,
|
||||
complianceResult: mongoose.Schema.Types.Mixed,
|
||||
processingTime: Number,
|
||||
timestamp: {
|
||||
type: Date,
|
||||
default: Date.now,
|
||||
index: true
|
||||
}
|
||||
}, {
|
||||
timestamps: true
|
||||
});
|
||||
|
||||
// Indexes
|
||||
policyViolationSchema.index({ timestamp: -1 });
|
||||
policyViolationSchema.index({ policyId: 1, timestamp: -1 });
|
||||
policyViolationSchema.index({ 'violations.type': 1 });
|
||||
|
||||
// TTL index to auto-delete old violations after 90 days
|
||||
policyViolationSchema.index({ timestamp: 1 }, { expireAfterSeconds: 7776000 });
|
||||
|
||||
// Multi-tenant indexes
|
||||
policyViolationSchema.index({ tenantId: 1, timestamp: -1 });
|
||||
policyViolationSchema.index({ tenantId: 1, policyId: 1, timestamp: -1 });
|
||||
policyViolationSchema.index({ tenantId: 1, 'violations.type': 1 });
|
||||
policyViolationSchema.index({ tenantId: 1, timestamp: 1 }, { expireAfterSeconds: 7776000 });
|
||||
|
||||
export const PolicyViolation = mongoose.model('PolicyViolation', policyViolationSchema);
|
||||
@@ -0,0 +1,48 @@
|
||||
import mongoose from 'mongoose';
|
||||
|
||||
const rateLimitViolationSchema = new mongoose.Schema({
|
||||
// Multi-tenant support
|
||||
tenantId: {
|
||||
type: mongoose.Schema.Types.ObjectId,
|
||||
ref: 'Tenant',
|
||||
required: true,
|
||||
index: true
|
||||
},
|
||||
limitType: {
|
||||
type: String,
|
||||
required: true
|
||||
},
|
||||
identifier: {
|
||||
type: String,
|
||||
required: true
|
||||
},
|
||||
tier: String,
|
||||
cost: {
|
||||
type: Number,
|
||||
default: 1
|
||||
},
|
||||
msBeforeNext: Number,
|
||||
timestamp: {
|
||||
type: Date,
|
||||
default: Date.now,
|
||||
index: true
|
||||
}
|
||||
}, {
|
||||
timestamps: true
|
||||
});
|
||||
|
||||
// Indexes
|
||||
rateLimitViolationSchema.index({ timestamp: -1 });
|
||||
rateLimitViolationSchema.index({ limitType: 1, identifier: 1 });
|
||||
rateLimitViolationSchema.index({ identifier: 1, timestamp: -1 });
|
||||
|
||||
// TTL index to auto-delete old violations after 30 days
|
||||
rateLimitViolationSchema.index({ timestamp: 1 }, { expireAfterSeconds: 2592000 });
|
||||
|
||||
// Multi-tenant indexes
|
||||
rateLimitViolationSchema.index({ tenantId: 1, timestamp: -1 });
|
||||
rateLimitViolationSchema.index({ tenantId: 1, limitType: 1, identifier: 1 });
|
||||
rateLimitViolationSchema.index({ tenantId: 1, identifier: 1, timestamp: -1 });
|
||||
rateLimitViolationSchema.index({ tenantId: 1, timestamp: 1 }, { expireAfterSeconds: 2592000 });
|
||||
|
||||
export const RateLimitViolation = mongoose.model('RateLimitViolation', rateLimitViolationSchema);
|
||||
557
marketing-agent/services/safety-guard/src/routes/index.js
Normal file
557
marketing-agent/services/safety-guard/src/routes/index.js
Normal file
@@ -0,0 +1,557 @@
|
||||
import Joi from '@hapi/joi';
|
||||
import { ModerationService } from '../services/ModerationService.js';
|
||||
import { ComplianceService } from '../services/ComplianceService.js';
|
||||
import { RateLimiterService } from '../services/RateLimiterService.js';
|
||||
import { ContentPolicyService } from '../services/ContentPolicyService.js';
|
||||
import { logger } from '../utils/logger.js';
|
||||
|
||||
const moderationService = ModerationService.getInstance();
|
||||
const complianceService = ComplianceService.getInstance();
|
||||
const rateLimiterService = RateLimiterService.getInstance();
|
||||
const contentPolicyService = ContentPolicyService.getInstance();
|
||||
|
||||
export default [
|
||||
// Content Moderation Routes
|
||||
{
|
||||
method: 'POST',
|
||||
path: '/api/v1/moderate',
|
||||
options: {
|
||||
validate: {
|
||||
payload: Joi.object({
|
||||
content: Joi.string().required(),
|
||||
contentType: Joi.string().valid('text', 'image', 'video').default('text'),
|
||||
context: Joi.object({
|
||||
accountId: Joi.string(),
|
||||
chatId: Joi.string(),
|
||||
userId: Joi.string(),
|
||||
language: Joi.string()
|
||||
}).optional()
|
||||
})
|
||||
},
|
||||
handler: async (request, h) => {
|
||||
try {
|
||||
const { content } = request.payload;
|
||||
|
||||
const result = await moderationService.moderateContent(content);
|
||||
|
||||
return h.response({
|
||||
success: true,
|
||||
data: result
|
||||
});
|
||||
} catch (error) {
|
||||
logger.error('Moderation error:', error);
|
||||
return h.response({
|
||||
success: false,
|
||||
error: error.message
|
||||
}).code(500);
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
|
||||
{
|
||||
method: 'POST',
|
||||
path: '/api/v1/moderate/bulk',
|
||||
options: {
|
||||
validate: {
|
||||
payload: Joi.object({
|
||||
contents: Joi.array().items(Joi.string()).required(),
|
||||
contentType: Joi.string().valid('text', 'image', 'video').default('text')
|
||||
})
|
||||
},
|
||||
handler: async (request, h) => {
|
||||
try {
|
||||
const { contents } = request.payload;
|
||||
|
||||
const result = await moderationService.moderateBulkContent(contents);
|
||||
|
||||
return h.response({
|
||||
success: true,
|
||||
data: result
|
||||
});
|
||||
} catch (error) {
|
||||
logger.error('Bulk moderation error:', error);
|
||||
return h.response({
|
||||
success: false,
|
||||
error: error.message
|
||||
}).code(500);
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
|
||||
// Compliance Routes
|
||||
{
|
||||
method: 'POST',
|
||||
path: '/api/v1/compliance/check',
|
||||
options: {
|
||||
validate: {
|
||||
payload: Joi.object({
|
||||
action: Joi.string().required(),
|
||||
content: Joi.string().required(),
|
||||
targetRegion: Joi.string().required(),
|
||||
targetAudience: Joi.object({
|
||||
minAge: Joi.number().min(0),
|
||||
segments: Joi.array().items(Joi.string())
|
||||
}).optional(),
|
||||
metadata: Joi.object().optional()
|
||||
})
|
||||
},
|
||||
handler: async (request, h) => {
|
||||
try {
|
||||
const result = await complianceService.checkCompliance(request.payload);
|
||||
|
||||
return h.response({
|
||||
success: true,
|
||||
data: result
|
||||
});
|
||||
} catch (error) {
|
||||
logger.error('Compliance check error:', error);
|
||||
return h.response({
|
||||
success: false,
|
||||
error: error.message
|
||||
}).code(500);
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
|
||||
{
|
||||
method: 'POST',
|
||||
path: '/api/v1/compliance/consent/validate',
|
||||
options: {
|
||||
validate: {
|
||||
payload: Joi.object({
|
||||
userId: Joi.string().required(),
|
||||
purpose: Joi.string().required(),
|
||||
region: Joi.string().required()
|
||||
})
|
||||
},
|
||||
handler: async (request, h) => {
|
||||
try {
|
||||
const { userId, purpose, region } = request.payload;
|
||||
|
||||
const result = await complianceService.validateConsent(userId, purpose, region);
|
||||
|
||||
return h.response({
|
||||
success: true,
|
||||
data: result
|
||||
});
|
||||
} catch (error) {
|
||||
logger.error('Consent validation error:', error);
|
||||
return h.response({
|
||||
success: false,
|
||||
error: error.message
|
||||
}).code(500);
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
|
||||
{
|
||||
method: 'GET',
|
||||
path: '/api/v1/compliance/report',
|
||||
options: {
|
||||
validate: {
|
||||
query: Joi.object({
|
||||
period: Joi.string().valid('24h', '7d', '30d').default('7d')
|
||||
})
|
||||
},
|
||||
handler: async (request, h) => {
|
||||
try {
|
||||
const { period } = request.query;
|
||||
|
||||
const report = await complianceService.getComplianceReport(period);
|
||||
|
||||
return h.response({
|
||||
success: true,
|
||||
data: report
|
||||
});
|
||||
} catch (error) {
|
||||
logger.error('Compliance report error:', error);
|
||||
return h.response({
|
||||
success: false,
|
||||
error: error.message
|
||||
}).code(500);
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
|
||||
// Rate Limiting Routes
|
||||
{
|
||||
method: 'POST',
|
||||
path: '/api/v1/ratelimit/check',
|
||||
options: {
|
||||
validate: {
|
||||
payload: Joi.object({
|
||||
limitType: Joi.string().required(),
|
||||
identifier: Joi.string().required(),
|
||||
tier: Joi.string().valid('free', 'basic', 'pro', 'enterprise').optional(),
|
||||
cost: Joi.number().min(1).default(1)
|
||||
})
|
||||
},
|
||||
handler: async (request, h) => {
|
||||
try {
|
||||
const { limitType, identifier, tier, cost } = request.payload;
|
||||
|
||||
const result = await rateLimiterService.checkRateLimit(
|
||||
limitType,
|
||||
identifier,
|
||||
{ tier, cost }
|
||||
);
|
||||
|
||||
return h.response({
|
||||
success: true,
|
||||
data: result
|
||||
}).code(result.allowed ? 200 : 429);
|
||||
} catch (error) {
|
||||
logger.error('Rate limit check error:', error);
|
||||
return h.response({
|
||||
success: false,
|
||||
error: error.message
|
||||
}).code(500);
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
|
||||
{
|
||||
method: 'POST',
|
||||
path: '/api/v1/ratelimit/check-multiple',
|
||||
options: {
|
||||
validate: {
|
||||
payload: Joi.object({
|
||||
limits: Joi.array().items(Joi.string()).required(),
|
||||
identifier: Joi.string().required(),
|
||||
tier: Joi.string().valid('free', 'basic', 'pro', 'enterprise').optional()
|
||||
})
|
||||
},
|
||||
handler: async (request, h) => {
|
||||
try {
|
||||
const { limits, identifier, tier } = request.payload;
|
||||
|
||||
const result = await rateLimiterService.checkMultipleRateLimits(
|
||||
limits,
|
||||
identifier,
|
||||
{ tier }
|
||||
);
|
||||
|
||||
return h.response({
|
||||
success: true,
|
||||
data: result
|
||||
}).code(result.allowed ? 200 : 429);
|
||||
} catch (error) {
|
||||
logger.error('Multiple rate limit check error:', error);
|
||||
return h.response({
|
||||
success: false,
|
||||
error: error.message
|
||||
}).code(500);
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
|
||||
{
|
||||
method: 'GET',
|
||||
path: '/api/v1/ratelimit/status',
|
||||
options: {
|
||||
validate: {
|
||||
query: Joi.object({
|
||||
limitTypes: Joi.array().items(Joi.string()).single().required(),
|
||||
identifier: Joi.string().required(),
|
||||
tier: Joi.string().valid('free', 'basic', 'pro', 'enterprise').optional()
|
||||
})
|
||||
},
|
||||
handler: async (request, h) => {
|
||||
try {
|
||||
const { limitTypes, identifier, tier } = request.query;
|
||||
|
||||
const status = await rateLimiterService.getRateLimitStatus(
|
||||
limitTypes,
|
||||
identifier,
|
||||
tier
|
||||
);
|
||||
|
||||
return h.response({
|
||||
success: true,
|
||||
data: status
|
||||
});
|
||||
} catch (error) {
|
||||
logger.error('Rate limit status error:', error);
|
||||
return h.response({
|
||||
success: false,
|
||||
error: error.message
|
||||
}).code(500);
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
|
||||
{
|
||||
method: 'POST',
|
||||
path: '/api/v1/ratelimit/reset',
|
||||
options: {
|
||||
validate: {
|
||||
payload: Joi.object({
|
||||
limitType: Joi.string().required(),
|
||||
identifier: Joi.string().required()
|
||||
})
|
||||
},
|
||||
handler: async (request, h) => {
|
||||
try {
|
||||
const { limitType, identifier } = request.payload;
|
||||
|
||||
const result = await rateLimiterService.resetRateLimit(limitType, identifier);
|
||||
|
||||
return h.response({
|
||||
success: true,
|
||||
data: result
|
||||
});
|
||||
} catch (error) {
|
||||
logger.error('Rate limit reset error:', error);
|
||||
return h.response({
|
||||
success: false,
|
||||
error: error.message
|
||||
}).code(500);
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
|
||||
{
|
||||
method: 'GET',
|
||||
path: '/api/v1/ratelimit/violations',
|
||||
options: {
|
||||
validate: {
|
||||
query: Joi.object({
|
||||
period: Joi.string().valid('1h', '24h', '7d').default('24h')
|
||||
})
|
||||
},
|
||||
handler: async (request, h) => {
|
||||
try {
|
||||
const { period } = request.query;
|
||||
|
||||
const report = await rateLimiterService.getViolationReport(period);
|
||||
|
||||
return h.response({
|
||||
success: true,
|
||||
data: report
|
||||
});
|
||||
} catch (error) {
|
||||
logger.error('Rate limit violation report error:', error);
|
||||
return h.response({
|
||||
success: false,
|
||||
error: error.message
|
||||
}).code(500);
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
|
||||
// Content Policy Routes
|
||||
{
|
||||
method: 'POST',
|
||||
path: '/api/v1/policy/validate',
|
||||
options: {
|
||||
validate: {
|
||||
payload: Joi.object({
|
||||
content: Joi.string().required(),
|
||||
policyId: Joi.string().default('default'),
|
||||
context: Joi.object({
|
||||
contentType: Joi.string().valid('text', 'image', 'video', 'document'),
|
||||
mediaSize: Joi.number().min(0),
|
||||
targetRegion: Joi.string(),
|
||||
targetAudience: Joi.object(),
|
||||
action: Joi.string(),
|
||||
accountId: Joi.string(),
|
||||
metadata: Joi.object()
|
||||
}).optional()
|
||||
})
|
||||
},
|
||||
handler: async (request, h) => {
|
||||
try {
|
||||
const { content, policyId, context } = request.payload;
|
||||
|
||||
const result = await contentPolicyService.validateContent(
|
||||
content,
|
||||
policyId,
|
||||
context || {}
|
||||
);
|
||||
|
||||
return h.response({
|
||||
success: true,
|
||||
data: result
|
||||
});
|
||||
} catch (error) {
|
||||
logger.error('Policy validation error:', error);
|
||||
return h.response({
|
||||
success: false,
|
||||
error: error.message
|
||||
}).code(500);
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
|
||||
{
|
||||
method: 'PUT',
|
||||
path: '/api/v1/policy/{policyId}',
|
||||
options: {
|
||||
validate: {
|
||||
params: Joi.object({
|
||||
policyId: Joi.string().required()
|
||||
}),
|
||||
payload: Joi.object({
|
||||
name: Joi.string(),
|
||||
description: Joi.string(),
|
||||
rules: Joi.object(),
|
||||
active: Joi.boolean()
|
||||
})
|
||||
},
|
||||
handler: async (request, h) => {
|
||||
try {
|
||||
const { policyId } = request.params;
|
||||
const updates = request.payload;
|
||||
|
||||
const policy = await contentPolicyService.updatePolicy(policyId, updates);
|
||||
|
||||
return h.response({
|
||||
success: true,
|
||||
data: policy
|
||||
});
|
||||
} catch (error) {
|
||||
logger.error('Policy update error:', error);
|
||||
return h.response({
|
||||
success: false,
|
||||
error: error.message
|
||||
}).code(500);
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
|
||||
{
|
||||
method: 'GET',
|
||||
path: '/api/v1/policy/violations',
|
||||
options: {
|
||||
validate: {
|
||||
query: Joi.object({
|
||||
period: Joi.string().valid('24h', '7d', '30d').default('7d')
|
||||
})
|
||||
},
|
||||
handler: async (request, h) => {
|
||||
try {
|
||||
const { period } = request.query;
|
||||
|
||||
const report = await contentPolicyService.getPolicyViolationReport(period);
|
||||
|
||||
return h.response({
|
||||
success: true,
|
||||
data: report
|
||||
});
|
||||
} catch (error) {
|
||||
logger.error('Policy violation report error:', error);
|
||||
return h.response({
|
||||
success: false,
|
||||
error: error.message
|
||||
}).code(500);
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
|
||||
// Combined Safety Check
|
||||
{
|
||||
method: 'POST',
|
||||
path: '/api/v1/safety/check',
|
||||
options: {
|
||||
validate: {
|
||||
payload: Joi.object({
|
||||
content: Joi.string().required(),
|
||||
action: Joi.string().required(),
|
||||
context: Joi.object({
|
||||
accountId: Joi.string().required(),
|
||||
targetRegion: Joi.string().required(),
|
||||
targetAudience: Joi.object(),
|
||||
contentType: Joi.string(),
|
||||
mediaSize: Joi.number(),
|
||||
tier: Joi.string(),
|
||||
policyId: Joi.string()
|
||||
}).required()
|
||||
})
|
||||
},
|
||||
handler: async (request, h) => {
|
||||
try {
|
||||
const { content, action, context } = request.payload;
|
||||
const startTime = Date.now();
|
||||
|
||||
// Run all safety checks in parallel
|
||||
const [moderation, compliance, rateLimit, policy] = await Promise.all([
|
||||
moderationService.moderateContent(content),
|
||||
complianceService.checkCompliance({
|
||||
action,
|
||||
content,
|
||||
targetRegion: context.targetRegion,
|
||||
targetAudience: context.targetAudience || {},
|
||||
metadata: context
|
||||
}),
|
||||
rateLimiterService.checkMultipleRateLimits(
|
||||
['message:minute', 'message:hour', 'message:day'],
|
||||
context.accountId,
|
||||
{ tier: context.tier }
|
||||
),
|
||||
contentPolicyService.validateContent(
|
||||
content,
|
||||
context.policyId || 'default',
|
||||
context
|
||||
)
|
||||
]);
|
||||
|
||||
const approved = moderation.approved &&
|
||||
compliance.compliant &&
|
||||
rateLimit.allowed &&
|
||||
policy.approved;
|
||||
|
||||
const result = {
|
||||
approved,
|
||||
checks: {
|
||||
moderation: {
|
||||
approved: moderation.approved,
|
||||
riskScore: moderation.riskScore,
|
||||
violations: moderation.violations
|
||||
},
|
||||
compliance: {
|
||||
compliant: compliance.compliant,
|
||||
violations: compliance.violations,
|
||||
warnings: compliance.warnings
|
||||
},
|
||||
rateLimit: {
|
||||
allowed: rateLimit.allowed,
|
||||
failedLimit: rateLimit.failedLimit,
|
||||
results: rateLimit.results
|
||||
},
|
||||
policy: {
|
||||
approved: policy.approved,
|
||||
violations: policy.violations,
|
||||
warnings: policy.warnings
|
||||
}
|
||||
},
|
||||
processingTime: Date.now() - startTime
|
||||
};
|
||||
|
||||
return h.response({
|
||||
success: true,
|
||||
data: result
|
||||
}).code(approved ? 200 : 403);
|
||||
} catch (error) {
|
||||
logger.error('Safety check error:', error);
|
||||
return h.response({
|
||||
success: false,
|
||||
error: error.message
|
||||
}).code(500);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
];
|
||||
@@ -0,0 +1,427 @@
|
||||
import { logger } from '../utils/logger.js';
|
||||
import { ComplianceRule } from '../models/ComplianceRule.js';
|
||||
import { ComplianceCheck } from '../models/ComplianceCheck.js';
|
||||
import { RedisClient } from '../config/redis.js';
|
||||
|
||||
export class ComplianceService {
|
||||
constructor() {
|
||||
this.redis = null;
|
||||
this.rules = new Map();
|
||||
this.regions = new Map();
|
||||
}
|
||||
|
||||
static getInstance() {
|
||||
if (!ComplianceService.instance) {
|
||||
ComplianceService.instance = new ComplianceService();
|
||||
ComplianceService.instance.initialize();
|
||||
}
|
||||
return ComplianceService.instance;
|
||||
}
|
||||
|
||||
async initialize() {
|
||||
this.redis = RedisClient.getInstance();
|
||||
|
||||
// Load compliance rules
|
||||
await this.loadComplianceRules();
|
||||
|
||||
// Initialize region-specific rules
|
||||
this.initializeRegionRules();
|
||||
|
||||
logger.info('Compliance service initialized');
|
||||
}
|
||||
|
||||
async loadComplianceRules() {
|
||||
try {
|
||||
const rules = await ComplianceRule.find({ active: true });
|
||||
|
||||
for (const rule of rules) {
|
||||
this.rules.set(rule.ruleId, rule);
|
||||
}
|
||||
|
||||
logger.info(`Loaded ${rules.length} compliance rules`);
|
||||
} catch (error) {
|
||||
logger.error('Failed to load compliance rules:', error);
|
||||
}
|
||||
}
|
||||
|
||||
initializeRegionRules() {
|
||||
// GDPR (European Union)
|
||||
this.regions.set('EU', {
|
||||
dataRetention: 90, // days
|
||||
consentRequired: true,
|
||||
rightToDelete: true,
|
||||
dataPortability: true,
|
||||
regulations: ['GDPR'],
|
||||
restrictions: {
|
||||
minAge: 16,
|
||||
cookieConsent: true,
|
||||
explicitOptIn: true
|
||||
}
|
||||
});
|
||||
|
||||
// CCPA (California)
|
||||
this.regions.set('CA', {
|
||||
dataRetention: 365,
|
||||
consentRequired: true,
|
||||
rightToDelete: true,
|
||||
dataPortability: true,
|
||||
regulations: ['CCPA'],
|
||||
restrictions: {
|
||||
minAge: 13,
|
||||
doNotSell: true,
|
||||
privacyNotice: true
|
||||
}
|
||||
});
|
||||
|
||||
// LGPD (Brazil)
|
||||
this.regions.set('BR', {
|
||||
dataRetention: 180,
|
||||
consentRequired: true,
|
||||
rightToDelete: true,
|
||||
dataPortability: true,
|
||||
regulations: ['LGPD'],
|
||||
restrictions: {
|
||||
minAge: 18,
|
||||
dataProtectionOfficer: true
|
||||
}
|
||||
});
|
||||
|
||||
// POPIA (South Africa)
|
||||
this.regions.set('ZA', {
|
||||
dataRetention: 365,
|
||||
consentRequired: true,
|
||||
rightToDelete: true,
|
||||
dataPortability: false,
|
||||
regulations: ['POPIA'],
|
||||
restrictions: {
|
||||
minAge: 18,
|
||||
crossBorderTransfer: 'restricted'
|
||||
}
|
||||
});
|
||||
|
||||
// Default (Rest of World)
|
||||
this.regions.set('DEFAULT', {
|
||||
dataRetention: 365,
|
||||
consentRequired: false,
|
||||
rightToDelete: false,
|
||||
dataPortability: false,
|
||||
regulations: [],
|
||||
restrictions: {
|
||||
minAge: 13
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
async checkCompliance(params) {
|
||||
const {
|
||||
action,
|
||||
content,
|
||||
targetRegion,
|
||||
targetAudience,
|
||||
metadata = {}
|
||||
} = params;
|
||||
|
||||
const startTime = Date.now();
|
||||
const violations = [];
|
||||
const warnings = [];
|
||||
|
||||
try {
|
||||
// Get region-specific rules
|
||||
const regionRules = this.regions.get(targetRegion) || this.regions.get('DEFAULT');
|
||||
|
||||
// Check age restrictions
|
||||
if (targetAudience.minAge && targetAudience.minAge < regionRules.restrictions.minAge) {
|
||||
violations.push({
|
||||
type: 'age_restriction',
|
||||
severity: 'high',
|
||||
message: `Minimum age ${regionRules.restrictions.minAge} required for ${targetRegion}`,
|
||||
regulation: regionRules.regulations[0]
|
||||
});
|
||||
}
|
||||
|
||||
// Check consent requirements
|
||||
if (regionRules.consentRequired && !metadata.hasConsent) {
|
||||
violations.push({
|
||||
type: 'consent_missing',
|
||||
severity: 'high',
|
||||
message: 'User consent required for this region',
|
||||
regulation: regionRules.regulations[0]
|
||||
});
|
||||
}
|
||||
|
||||
// Check data retention
|
||||
if (metadata.retentionDays && metadata.retentionDays > regionRules.dataRetention) {
|
||||
warnings.push({
|
||||
type: 'data_retention',
|
||||
severity: 'medium',
|
||||
message: `Data retention exceeds ${regionRules.dataRetention} days limit`,
|
||||
regulation: regionRules.regulations[0]
|
||||
});
|
||||
}
|
||||
|
||||
// Check content-specific rules
|
||||
const contentChecks = await this.checkContentCompliance(content, targetRegion);
|
||||
violations.push(...contentChecks.violations);
|
||||
warnings.push(...contentChecks.warnings);
|
||||
|
||||
// Check action-specific rules
|
||||
const actionChecks = await this.checkActionCompliance(action, targetRegion, metadata);
|
||||
violations.push(...actionChecks.violations);
|
||||
warnings.push(...actionChecks.warnings);
|
||||
|
||||
// Log compliance check
|
||||
await this.logComplianceCheck({
|
||||
action,
|
||||
targetRegion,
|
||||
violations,
|
||||
warnings,
|
||||
processingTime: Date.now() - startTime
|
||||
});
|
||||
|
||||
return {
|
||||
compliant: violations.length === 0,
|
||||
violations,
|
||||
warnings,
|
||||
regionRules,
|
||||
processingTime: Date.now() - startTime
|
||||
};
|
||||
} catch (error) {
|
||||
logger.error('Compliance check error:', error);
|
||||
return {
|
||||
compliant: true, // Fail open
|
||||
error: error.message
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
async checkContentCompliance(content, region) {
|
||||
const violations = [];
|
||||
const warnings = [];
|
||||
|
||||
// Check for regulated content types
|
||||
const regulatedTerms = {
|
||||
'EU': [
|
||||
{ term: 'personal data', type: 'gdpr_sensitive' },
|
||||
{ term: 'health', type: 'special_category' },
|
||||
{ term: 'political', type: 'special_category' }
|
||||
],
|
||||
'CA': [
|
||||
{ term: 'sale of data', type: 'ccpa_restricted' },
|
||||
{ term: 'financial', type: 'sensitive' }
|
||||
]
|
||||
};
|
||||
|
||||
const regionTerms = regulatedTerms[region] || [];
|
||||
const contentLower = content.toLowerCase();
|
||||
|
||||
for (const { term, type } of regionTerms) {
|
||||
if (contentLower.includes(term)) {
|
||||
warnings.push({
|
||||
type: 'regulated_content',
|
||||
severity: 'medium',
|
||||
message: `Content contains regulated term: ${term}`,
|
||||
category: type
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Check for marketing-specific compliance
|
||||
if (region === 'EU' || region === 'CA') {
|
||||
if (!content.includes('unsubscribe') && !content.includes('opt-out')) {
|
||||
violations.push({
|
||||
type: 'missing_opt_out',
|
||||
severity: 'high',
|
||||
message: 'Marketing messages must include opt-out option'
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return { violations, warnings };
|
||||
}
|
||||
|
||||
async checkActionCompliance(action, region, metadata) {
|
||||
const violations = [];
|
||||
const warnings = [];
|
||||
|
||||
// Check action-specific rules
|
||||
switch (action) {
|
||||
case 'bulk_message':
|
||||
if (region === 'EU' && !metadata.doubleOptIn) {
|
||||
violations.push({
|
||||
type: 'double_opt_in',
|
||||
severity: 'high',
|
||||
message: 'Double opt-in required for bulk messaging in EU'
|
||||
});
|
||||
}
|
||||
break;
|
||||
|
||||
case 'data_export':
|
||||
const regionRules = this.regions.get(region);
|
||||
if (!regionRules.dataPortability) {
|
||||
warnings.push({
|
||||
type: 'data_portability',
|
||||
severity: 'low',
|
||||
message: 'Data portability not required in this region'
|
||||
});
|
||||
}
|
||||
break;
|
||||
|
||||
case 'user_deletion':
|
||||
const rules = this.regions.get(region);
|
||||
if (rules.rightToDelete && !metadata.deletionScheduled) {
|
||||
violations.push({
|
||||
type: 'right_to_delete',
|
||||
severity: 'high',
|
||||
message: 'User deletion must be scheduled within required timeframe'
|
||||
});
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
// Check rate limits by region
|
||||
const rateLimits = {
|
||||
'EU': { messagesPerDay: 100, messagesPerHour: 20 },
|
||||
'CA': { messagesPerDay: 200, messagesPerHour: 30 },
|
||||
'DEFAULT': { messagesPerDay: 500, messagesPerHour: 50 }
|
||||
};
|
||||
|
||||
const limits = rateLimits[region] || rateLimits['DEFAULT'];
|
||||
|
||||
if (metadata.messageCount) {
|
||||
if (metadata.messageCount.daily > limits.messagesPerDay) {
|
||||
violations.push({
|
||||
type: 'rate_limit_daily',
|
||||
severity: 'medium',
|
||||
message: `Daily message limit exceeded: ${limits.messagesPerDay}`
|
||||
});
|
||||
}
|
||||
if (metadata.messageCount.hourly > limits.messagesPerHour) {
|
||||
violations.push({
|
||||
type: 'rate_limit_hourly',
|
||||
severity: 'medium',
|
||||
message: `Hourly message limit exceeded: ${limits.messagesPerHour}`
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return { violations, warnings };
|
||||
}
|
||||
|
||||
async validateConsent(userId, purpose, region) {
|
||||
const cacheKey = `consent:${userId}:${purpose}:${region}`;
|
||||
|
||||
// Check cache
|
||||
const cached = await this.redis.get(cacheKey);
|
||||
if (cached !== null) {
|
||||
return cached;
|
||||
}
|
||||
|
||||
// Check database
|
||||
const consent = await this.checkUserConsent(userId, purpose, region);
|
||||
|
||||
// Cache for 1 hour
|
||||
await this.redis.setWithExpiry(cacheKey, consent, 3600);
|
||||
|
||||
return consent;
|
||||
}
|
||||
|
||||
async checkUserConsent(userId, purpose, region) {
|
||||
// This would check actual consent records
|
||||
// For now, return a mock implementation
|
||||
return {
|
||||
hasConsent: false,
|
||||
consentDate: null,
|
||||
expiryDate: null,
|
||||
purposes: [],
|
||||
region
|
||||
};
|
||||
}
|
||||
|
||||
async logComplianceCheck(checkData) {
|
||||
try {
|
||||
await ComplianceCheck.create({
|
||||
action: checkData.action,
|
||||
targetRegion: checkData.targetRegion,
|
||||
violations: checkData.violations,
|
||||
warnings: checkData.warnings,
|
||||
processingTime: checkData.processingTime,
|
||||
timestamp: new Date()
|
||||
});
|
||||
} catch (error) {
|
||||
logger.error('Failed to log compliance check:', error);
|
||||
}
|
||||
}
|
||||
|
||||
async getComplianceReport(period = '7d') {
|
||||
const since = new Date();
|
||||
|
||||
switch (period) {
|
||||
case '24h':
|
||||
since.setDate(since.getDate() - 1);
|
||||
break;
|
||||
case '7d':
|
||||
since.setDate(since.getDate() - 7);
|
||||
break;
|
||||
case '30d':
|
||||
since.setDate(since.getDate() - 30);
|
||||
break;
|
||||
}
|
||||
|
||||
const report = await ComplianceCheck.aggregate([
|
||||
{ $match: { timestamp: { $gte: since } } },
|
||||
{
|
||||
$group: {
|
||||
_id: {
|
||||
region: '$targetRegion',
|
||||
action: '$action'
|
||||
},
|
||||
totalChecks: { $sum: 1 },
|
||||
violations: { $sum: { $size: '$violations' } },
|
||||
warnings: { $sum: { $size: '$warnings' } }
|
||||
}
|
||||
},
|
||||
{
|
||||
$group: {
|
||||
_id: '$_id.region',
|
||||
actions: {
|
||||
$push: {
|
||||
action: '$_id.action',
|
||||
totalChecks: '$totalChecks',
|
||||
violations: '$violations',
|
||||
warnings: '$warnings'
|
||||
}
|
||||
},
|
||||
totalChecks: { $sum: '$totalChecks' },
|
||||
totalViolations: { $sum: '$violations' },
|
||||
totalWarnings: { $sum: '$warnings' }
|
||||
}
|
||||
}
|
||||
]);
|
||||
|
||||
return {
|
||||
period,
|
||||
since,
|
||||
regions: report
|
||||
};
|
||||
}
|
||||
|
||||
async updateComplianceRule(ruleId, updates) {
|
||||
try {
|
||||
const rule = await ComplianceRule.findOneAndUpdate(
|
||||
{ ruleId },
|
||||
updates,
|
||||
{ new: true }
|
||||
);
|
||||
|
||||
if (rule) {
|
||||
this.rules.set(ruleId, rule);
|
||||
logger.info(`Updated compliance rule: ${ruleId}`);
|
||||
}
|
||||
|
||||
return rule;
|
||||
} catch (error) {
|
||||
logger.error('Failed to update compliance rule:', error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,558 @@
|
||||
import { logger } from '../utils/logger.js';
|
||||
import { ContentPolicy } from '../models/ContentPolicy.js';
|
||||
import { PolicyViolation } from '../models/PolicyViolation.js';
|
||||
import { ModerationService } from './ModerationService.js';
|
||||
import { ComplianceService } from './ComplianceService.js';
|
||||
|
||||
export class ContentPolicyService {
|
||||
constructor() {
|
||||
this.policies = new Map();
|
||||
this.moderationService = null;
|
||||
this.complianceService = null;
|
||||
}
|
||||
|
||||
static getInstance() {
|
||||
if (!ContentPolicyService.instance) {
|
||||
ContentPolicyService.instance = new ContentPolicyService();
|
||||
ContentPolicyService.instance.initialize();
|
||||
}
|
||||
return ContentPolicyService.instance;
|
||||
}
|
||||
|
||||
async initialize() {
|
||||
this.moderationService = ModerationService.getInstance();
|
||||
this.complianceService = ComplianceService.getInstance();
|
||||
|
||||
// Load content policies
|
||||
await this.loadPolicies();
|
||||
|
||||
logger.info('Content policy service initialized');
|
||||
}
|
||||
|
||||
async loadPolicies() {
|
||||
try {
|
||||
const policies = await ContentPolicy.find({ active: true });
|
||||
|
||||
for (const policy of policies) {
|
||||
this.policies.set(policy.policyId, policy);
|
||||
}
|
||||
|
||||
// Create default policies if none exist
|
||||
if (this.policies.size === 0) {
|
||||
await this.createDefaultPolicies();
|
||||
}
|
||||
|
||||
logger.info(`Loaded ${this.policies.size} content policies`);
|
||||
} catch (error) {
|
||||
logger.error('Failed to load content policies:', error);
|
||||
}
|
||||
}
|
||||
|
||||
async createDefaultPolicies() {
|
||||
const defaultPolicies = [
|
||||
{
|
||||
policyId: 'default',
|
||||
name: 'Default Content Policy',
|
||||
description: 'Standard content policy for general use',
|
||||
rules: {
|
||||
maxMessageLength: 4096,
|
||||
minMessageLength: 1,
|
||||
allowedContentTypes: ['text', 'image', 'video', 'document'],
|
||||
prohibitedPatterns: [],
|
||||
maxMediaSize: 50 * 1024 * 1024, // 50MB
|
||||
maxRiskScore: 0.7,
|
||||
requireModeration: true,
|
||||
requireCompliance: true
|
||||
},
|
||||
active: true
|
||||
},
|
||||
{
|
||||
policyId: 'strict',
|
||||
name: 'Strict Content Policy',
|
||||
description: 'Enhanced content restrictions for sensitive audiences',
|
||||
rules: {
|
||||
maxMessageLength: 1000,
|
||||
minMessageLength: 10,
|
||||
allowedContentTypes: ['text'],
|
||||
prohibitedPatterns: [
|
||||
'http[s]?://[^\\s]+', // URLs
|
||||
'\\b\\d{3,}\\b', // Phone numbers
|
||||
'[\\w._%+-]+@[\\w.-]+\\.[A-Za-z]{2,}' // Emails
|
||||
],
|
||||
maxMediaSize: 10 * 1024 * 1024, // 10MB
|
||||
maxRiskScore: 0.3,
|
||||
requireModeration: true,
|
||||
requireCompliance: true,
|
||||
additionalChecks: ['duplicate_detection', 'spam_detection']
|
||||
},
|
||||
active: true
|
||||
},
|
||||
{
|
||||
policyId: 'marketing',
|
||||
name: 'Marketing Content Policy',
|
||||
description: 'Policy optimized for marketing campaigns',
|
||||
rules: {
|
||||
maxMessageLength: 2000,
|
||||
minMessageLength: 50,
|
||||
allowedContentTypes: ['text', 'image', 'video'],
|
||||
prohibitedPatterns: [],
|
||||
maxMediaSize: 25 * 1024 * 1024, // 25MB
|
||||
maxRiskScore: 0.5,
|
||||
requireModeration: true,
|
||||
requireCompliance: true,
|
||||
requireOptOut: true,
|
||||
maxLinksPerMessage: 2,
|
||||
requireCTA: true
|
||||
},
|
||||
active: true
|
||||
}
|
||||
];
|
||||
|
||||
for (const policyData of defaultPolicies) {
|
||||
const policy = await ContentPolicy.create(policyData);
|
||||
this.policies.set(policy.policyId, policy);
|
||||
}
|
||||
}
|
||||
|
||||
async validateContent(content, policyId = 'default', context = {}) {
|
||||
const startTime = Date.now();
|
||||
|
||||
try {
|
||||
// Get policy
|
||||
const policy = this.policies.get(policyId);
|
||||
if (!policy) {
|
||||
throw new Error(`Policy not found: ${policyId}`);
|
||||
}
|
||||
|
||||
const violations = [];
|
||||
const warnings = [];
|
||||
|
||||
// Basic content checks
|
||||
const basicChecks = this.performBasicChecks(content, policy.rules);
|
||||
violations.push(...basicChecks.violations);
|
||||
warnings.push(...basicChecks.warnings);
|
||||
|
||||
// Pattern matching
|
||||
const patternChecks = this.checkProhibitedPatterns(content, policy.rules);
|
||||
violations.push(...patternChecks.violations);
|
||||
|
||||
// Content type validation
|
||||
if (context.contentType) {
|
||||
const typeCheck = this.validateContentType(context.contentType, policy.rules);
|
||||
if (!typeCheck.allowed) {
|
||||
violations.push(typeCheck.violation);
|
||||
}
|
||||
}
|
||||
|
||||
// Media size validation
|
||||
if (context.mediaSize) {
|
||||
const sizeCheck = this.validateMediaSize(context.mediaSize, policy.rules);
|
||||
if (!sizeCheck.allowed) {
|
||||
violations.push(sizeCheck.violation);
|
||||
}
|
||||
}
|
||||
|
||||
// Moderation check
|
||||
let moderationResult = null;
|
||||
if (policy.rules.requireModeration) {
|
||||
moderationResult = await this.moderationService.moderateContent(content);
|
||||
if (!moderationResult.approved) {
|
||||
violations.push({
|
||||
type: 'moderation_failed',
|
||||
severity: 'high',
|
||||
message: 'Content failed moderation',
|
||||
details: moderationResult.violations
|
||||
});
|
||||
}
|
||||
if (moderationResult.riskScore > policy.rules.maxRiskScore) {
|
||||
violations.push({
|
||||
type: 'risk_score_exceeded',
|
||||
severity: 'high',
|
||||
message: `Risk score ${moderationResult.riskScore} exceeds limit ${policy.rules.maxRiskScore}`
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Compliance check
|
||||
let complianceResult = null;
|
||||
if (policy.rules.requireCompliance && context.targetRegion) {
|
||||
complianceResult = await this.complianceService.checkCompliance({
|
||||
action: context.action || 'message',
|
||||
content,
|
||||
targetRegion: context.targetRegion,
|
||||
targetAudience: context.targetAudience || {},
|
||||
metadata: context.metadata || {}
|
||||
});
|
||||
|
||||
if (!complianceResult.compliant) {
|
||||
violations.push(...complianceResult.violations.map(v => ({
|
||||
type: 'compliance_violation',
|
||||
severity: v.severity,
|
||||
message: v.message,
|
||||
regulation: v.regulation
|
||||
})));
|
||||
}
|
||||
warnings.push(...complianceResult.warnings);
|
||||
}
|
||||
|
||||
// Additional checks
|
||||
if (policy.rules.additionalChecks) {
|
||||
const additionalResults = await this.performAdditionalChecks(
|
||||
content,
|
||||
policy.rules.additionalChecks,
|
||||
context
|
||||
);
|
||||
violations.push(...additionalResults.violations);
|
||||
warnings.push(...additionalResults.warnings);
|
||||
}
|
||||
|
||||
// Log policy check
|
||||
await this.logPolicyCheck({
|
||||
policyId,
|
||||
content: content.substring(0, 200), // Log preview only
|
||||
violations,
|
||||
warnings,
|
||||
moderationResult,
|
||||
complianceResult,
|
||||
processingTime: Date.now() - startTime
|
||||
});
|
||||
|
||||
return {
|
||||
approved: violations.length === 0,
|
||||
violations,
|
||||
warnings,
|
||||
policy: {
|
||||
id: policy.policyId,
|
||||
name: policy.name
|
||||
},
|
||||
moderation: moderationResult,
|
||||
compliance: complianceResult,
|
||||
processingTime: Date.now() - startTime
|
||||
};
|
||||
} catch (error) {
|
||||
logger.error('Content validation error:', error);
|
||||
return {
|
||||
approved: false,
|
||||
error: error.message
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
performBasicChecks(content, rules) {
|
||||
const violations = [];
|
||||
const warnings = [];
|
||||
|
||||
// Length checks
|
||||
if (content.length < rules.minMessageLength) {
|
||||
violations.push({
|
||||
type: 'message_too_short',
|
||||
severity: 'medium',
|
||||
message: `Message length ${content.length} is below minimum ${rules.minMessageLength}`
|
||||
});
|
||||
}
|
||||
|
||||
if (content.length > rules.maxMessageLength) {
|
||||
violations.push({
|
||||
type: 'message_too_long',
|
||||
severity: 'medium',
|
||||
message: `Message length ${content.length} exceeds maximum ${rules.maxMessageLength}`
|
||||
});
|
||||
}
|
||||
|
||||
// Marketing-specific checks
|
||||
if (rules.requireOptOut && !content.toLowerCase().includes('opt') && !content.toLowerCase().includes('unsubscribe')) {
|
||||
violations.push({
|
||||
type: 'missing_opt_out',
|
||||
severity: 'high',
|
||||
message: 'Marketing messages must include opt-out option'
|
||||
});
|
||||
}
|
||||
|
||||
if (rules.requireCTA) {
|
||||
const ctaPatterns = ['click', 'join', 'sign up', 'register', 'buy', 'purchase', 'subscribe'];
|
||||
const hasCTA = ctaPatterns.some(pattern => content.toLowerCase().includes(pattern));
|
||||
if (!hasCTA) {
|
||||
warnings.push({
|
||||
type: 'missing_cta',
|
||||
severity: 'low',
|
||||
message: 'Marketing messages should include a clear call-to-action'
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Link checks
|
||||
if (rules.maxLinksPerMessage !== undefined) {
|
||||
const linkPattern = /https?:\/\/[^\s]+/g;
|
||||
const links = content.match(linkPattern) || [];
|
||||
if (links.length > rules.maxLinksPerMessage) {
|
||||
violations.push({
|
||||
type: 'too_many_links',
|
||||
severity: 'medium',
|
||||
message: `Message contains ${links.length} links, maximum allowed is ${rules.maxLinksPerMessage}`
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return { violations, warnings };
|
||||
}
|
||||
|
||||
checkProhibitedPatterns(content, rules) {
|
||||
const violations = [];
|
||||
|
||||
if (!rules.prohibitedPatterns || rules.prohibitedPatterns.length === 0) {
|
||||
return { violations };
|
||||
}
|
||||
|
||||
for (const pattern of rules.prohibitedPatterns) {
|
||||
try {
|
||||
const regex = new RegExp(pattern, 'gi');
|
||||
const matches = content.match(regex);
|
||||
|
||||
if (matches && matches.length > 0) {
|
||||
violations.push({
|
||||
type: 'prohibited_pattern',
|
||||
severity: 'high',
|
||||
message: `Content contains prohibited pattern`,
|
||||
matches: matches.slice(0, 5) // Limit to first 5 matches
|
||||
});
|
||||
}
|
||||
} catch (error) {
|
||||
logger.error(`Invalid regex pattern: ${pattern}`, error);
|
||||
}
|
||||
}
|
||||
|
||||
return { violations };
|
||||
}
|
||||
|
||||
validateContentType(contentType, rules) {
|
||||
if (!rules.allowedContentTypes || rules.allowedContentTypes.includes(contentType)) {
|
||||
return { allowed: true };
|
||||
}
|
||||
|
||||
return {
|
||||
allowed: false,
|
||||
violation: {
|
||||
type: 'invalid_content_type',
|
||||
severity: 'high',
|
||||
message: `Content type '${contentType}' is not allowed`
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
validateMediaSize(mediaSize, rules) {
|
||||
if (!rules.maxMediaSize || mediaSize <= rules.maxMediaSize) {
|
||||
return { allowed: true };
|
||||
}
|
||||
|
||||
return {
|
||||
allowed: false,
|
||||
violation: {
|
||||
type: 'media_size_exceeded',
|
||||
severity: 'medium',
|
||||
message: `Media size ${mediaSize} bytes exceeds maximum ${rules.maxMediaSize} bytes`
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
async performAdditionalChecks(content, checks, context) {
|
||||
const violations = [];
|
||||
const warnings = [];
|
||||
|
||||
for (const check of checks) {
|
||||
switch (check) {
|
||||
case 'duplicate_detection':
|
||||
const isDuplicate = await this.checkDuplicate(content, context);
|
||||
if (isDuplicate) {
|
||||
violations.push({
|
||||
type: 'duplicate_content',
|
||||
severity: 'medium',
|
||||
message: 'Duplicate content detected'
|
||||
});
|
||||
}
|
||||
break;
|
||||
|
||||
case 'spam_detection':
|
||||
const spamScore = await this.calculateSpamScore(content);
|
||||
if (spamScore > 0.7) {
|
||||
violations.push({
|
||||
type: 'spam_detected',
|
||||
severity: 'high',
|
||||
message: `Spam score ${spamScore} exceeds threshold`
|
||||
});
|
||||
}
|
||||
break;
|
||||
|
||||
case 'link_validation':
|
||||
const linkResults = await this.validateLinks(content);
|
||||
violations.push(...linkResults.violations);
|
||||
warnings.push(...linkResults.warnings);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return { violations, warnings };
|
||||
}
|
||||
|
||||
async checkDuplicate(content, context) {
|
||||
// Simple implementation - check for exact matches in recent messages
|
||||
// In production, use more sophisticated similarity algorithms
|
||||
const recentKey = `recent:${context.accountId || 'global'}`;
|
||||
const recentMessages = await this.redis.client.lrange(recentKey, 0, 99);
|
||||
|
||||
const contentHash = this.hashContent(content);
|
||||
const isDuplicate = recentMessages.includes(contentHash);
|
||||
|
||||
// Store this message hash
|
||||
await this.redis.client.lpush(recentKey, contentHash);
|
||||
await this.redis.client.ltrim(recentKey, 0, 99);
|
||||
await this.redis.client.expire(recentKey, 3600); // 1 hour
|
||||
|
||||
return isDuplicate;
|
||||
}
|
||||
|
||||
async calculateSpamScore(content) {
|
||||
// Simple spam scoring - in production use ML models
|
||||
let score = 0;
|
||||
|
||||
// Check for spam indicators
|
||||
const spamIndicators = [
|
||||
{ pattern: /FREE|WINNER|CONGRATULATIONS|CLAIM/gi, weight: 0.2 },
|
||||
{ pattern: /\$\d+|\d+%\s*OFF/gi, weight: 0.15 },
|
||||
{ pattern: /CLICK HERE|ACT NOW|LIMITED TIME/gi, weight: 0.15 },
|
||||
{ pattern: /!!!+|\?\?\?+/g, weight: 0.1 },
|
||||
{ pattern: /[A-Z]{10,}/g, weight: 0.1 } // Excessive caps
|
||||
];
|
||||
|
||||
for (const { pattern, weight } of spamIndicators) {
|
||||
const matches = content.match(pattern);
|
||||
if (matches) {
|
||||
score += weight * Math.min(matches.length, 3);
|
||||
}
|
||||
}
|
||||
|
||||
return Math.min(score, 1);
|
||||
}
|
||||
|
||||
async validateLinks(content) {
|
||||
const violations = [];
|
||||
const warnings = [];
|
||||
|
||||
const linkPattern = /https?:\/\/[^\s]+/g;
|
||||
const links = content.match(linkPattern) || [];
|
||||
|
||||
for (const link of links) {
|
||||
// Check against known malicious domains
|
||||
if (await this.isMaliciousLink(link)) {
|
||||
violations.push({
|
||||
type: 'malicious_link',
|
||||
severity: 'critical',
|
||||
message: `Malicious link detected: ${link}`
|
||||
});
|
||||
}
|
||||
|
||||
// Check for URL shorteners
|
||||
if (this.isURLShortener(link)) {
|
||||
warnings.push({
|
||||
type: 'url_shortener',
|
||||
severity: 'low',
|
||||
message: `URL shortener detected: ${link}`
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return { violations, warnings };
|
||||
}
|
||||
|
||||
async isMaliciousLink(url) {
|
||||
// In production, integrate with threat intelligence APIs
|
||||
const maliciousDomains = ['malicious.example.com', 'phishing.test.com'];
|
||||
return maliciousDomains.some(domain => url.includes(domain));
|
||||
}
|
||||
|
||||
isURLShortener(url) {
|
||||
const shorteners = ['bit.ly', 'tinyurl.com', 'goo.gl', 't.co', 'short.link'];
|
||||
return shorteners.some(shortener => url.includes(shortener));
|
||||
}
|
||||
|
||||
hashContent(content) {
|
||||
const crypto = require('crypto');
|
||||
return crypto.createHash('sha256').update(content).digest('hex');
|
||||
}
|
||||
|
||||
async logPolicyCheck(checkData) {
|
||||
try {
|
||||
await PolicyViolation.create({
|
||||
policyId: checkData.policyId,
|
||||
contentPreview: checkData.content,
|
||||
violations: checkData.violations,
|
||||
warnings: checkData.warnings,
|
||||
moderationResult: checkData.moderationResult,
|
||||
complianceResult: checkData.complianceResult,
|
||||
processingTime: checkData.processingTime,
|
||||
timestamp: new Date()
|
||||
});
|
||||
} catch (error) {
|
||||
logger.error('Failed to log policy check:', error);
|
||||
}
|
||||
}
|
||||
|
||||
async updatePolicy(policyId, updates) {
|
||||
try {
|
||||
const policy = await ContentPolicy.findOneAndUpdate(
|
||||
{ policyId },
|
||||
updates,
|
||||
{ new: true }
|
||||
);
|
||||
|
||||
if (policy) {
|
||||
this.policies.set(policyId, policy);
|
||||
logger.info(`Updated content policy: ${policyId}`);
|
||||
}
|
||||
|
||||
return policy;
|
||||
} catch (error) {
|
||||
logger.error('Failed to update content policy:', error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async getPolicyViolationReport(period = '7d') {
|
||||
const since = new Date();
|
||||
|
||||
switch (period) {
|
||||
case '24h':
|
||||
since.setDate(since.getDate() - 1);
|
||||
break;
|
||||
case '7d':
|
||||
since.setDate(since.getDate() - 7);
|
||||
break;
|
||||
case '30d':
|
||||
since.setDate(since.getDate() - 30);
|
||||
break;
|
||||
}
|
||||
|
||||
const report = await PolicyViolation.aggregate([
|
||||
{ $match: { timestamp: { $gte: since } } },
|
||||
{ $unwind: '$violations' },
|
||||
{
|
||||
$group: {
|
||||
_id: {
|
||||
policyId: '$policyId',
|
||||
violationType: '$violations.type'
|
||||
},
|
||||
count: { $sum: 1 },
|
||||
avgProcessingTime: { $avg: '$processingTime' }
|
||||
}
|
||||
},
|
||||
{
|
||||
$sort: { count: -1 }
|
||||
}
|
||||
]);
|
||||
|
||||
return {
|
||||
period,
|
||||
since,
|
||||
violations: report
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,390 @@
|
||||
import OpenAI from 'openai';
|
||||
import { LanguageServiceClient } from '@google-cloud/language';
|
||||
import BadWordsFilter from 'bad-words';
|
||||
import Sentiment from 'sentiment';
|
||||
import natural from 'natural';
|
||||
import { logger } from '../utils/logger.js';
|
||||
import { RedisClient } from '../config/redis.js';
|
||||
import { ContentViolation } from '../models/ContentViolation.js';
|
||||
|
||||
export class ModerationService {
|
||||
constructor() {
|
||||
this.openai = null;
|
||||
this.googleLanguage = null;
|
||||
this.badWordsFilter = null;
|
||||
this.sentiment = null;
|
||||
this.redis = null;
|
||||
this.tokenizer = null;
|
||||
}
|
||||
|
||||
static getInstance() {
|
||||
if (!ModerationService.instance) {
|
||||
ModerationService.instance = new ModerationService();
|
||||
ModerationService.instance.initialize();
|
||||
}
|
||||
return ModerationService.instance;
|
||||
}
|
||||
|
||||
initialize() {
|
||||
// Initialize OpenAI for advanced content moderation
|
||||
if (process.env.OPENAI_API_KEY) {
|
||||
this.openai = new OpenAI({
|
||||
apiKey: process.env.OPENAI_API_KEY
|
||||
});
|
||||
}
|
||||
|
||||
// Initialize Google Cloud Natural Language API
|
||||
if (process.env.GOOGLE_CLOUD_PROJECT) {
|
||||
this.googleLanguage = new LanguageServiceClient();
|
||||
}
|
||||
|
||||
// Initialize bad words filter
|
||||
this.badWordsFilter = new BadWordsFilter();
|
||||
|
||||
// Add custom bad words if needed
|
||||
const customBadWords = process.env.CUSTOM_BAD_WORDS?.split(',') || [];
|
||||
if (customBadWords.length > 0) {
|
||||
this.badWordsFilter.addWords(...customBadWords);
|
||||
}
|
||||
|
||||
// Initialize sentiment analyzer
|
||||
this.sentiment = new Sentiment();
|
||||
|
||||
// Initialize tokenizer
|
||||
this.tokenizer = new natural.WordTokenizer();
|
||||
|
||||
// Get Redis client
|
||||
this.redis = RedisClient.getInstance();
|
||||
|
||||
logger.info('Moderation service initialized');
|
||||
}
|
||||
|
||||
async moderateContent(content) {
|
||||
const startTime = Date.now();
|
||||
const cacheKey = `moderation:${this.hashContent(content)}`;
|
||||
|
||||
try {
|
||||
// Check cache first
|
||||
const cached = await this.redis.get(cacheKey);
|
||||
if (cached) {
|
||||
logger.debug('Moderation result from cache');
|
||||
return cached;
|
||||
}
|
||||
|
||||
// Run multiple moderation checks in parallel
|
||||
const [profanityCheck, sentimentAnalysis, aiModeration, toxicityCheck] = await Promise.all([
|
||||
this.checkProfanity(content),
|
||||
this.analyzeSentiment(content),
|
||||
this.aiContentModeration(content),
|
||||
this.checkToxicity(content)
|
||||
]);
|
||||
|
||||
// Combine results
|
||||
const result = {
|
||||
approved: true,
|
||||
violations: [],
|
||||
riskScore: 0,
|
||||
sentiment: sentimentAnalysis,
|
||||
checks: {
|
||||
profanity: profanityCheck,
|
||||
toxicity: toxicityCheck,
|
||||
ai: aiModeration
|
||||
},
|
||||
processingTime: Date.now() - startTime
|
||||
};
|
||||
|
||||
// Calculate overall risk score
|
||||
if (profanityCheck.detected) {
|
||||
result.riskScore += profanityCheck.severity * 0.3;
|
||||
result.violations.push({
|
||||
type: 'profanity',
|
||||
severity: profanityCheck.severity,
|
||||
details: profanityCheck.words
|
||||
});
|
||||
}
|
||||
|
||||
if (toxicityCheck.detected) {
|
||||
result.riskScore += toxicityCheck.score * 0.4;
|
||||
result.violations.push({
|
||||
type: 'toxicity',
|
||||
severity: toxicityCheck.score,
|
||||
categories: toxicityCheck.categories
|
||||
});
|
||||
}
|
||||
|
||||
if (aiModeration && !aiModeration.approved) {
|
||||
result.riskScore += aiModeration.riskScore * 0.3;
|
||||
result.violations.push(...aiModeration.violations);
|
||||
}
|
||||
|
||||
// Determine approval based on risk score
|
||||
result.approved = result.riskScore < 0.7;
|
||||
|
||||
// Cache result for 1 hour
|
||||
await this.redis.setWithExpiry(cacheKey, result, 3600);
|
||||
|
||||
// Log violations
|
||||
if (!result.approved) {
|
||||
await this.logViolation(content, result);
|
||||
}
|
||||
|
||||
return result;
|
||||
} catch (error) {
|
||||
logger.error('Moderation error:', error);
|
||||
// Return permissive result on error
|
||||
return {
|
||||
approved: true,
|
||||
violations: [],
|
||||
riskScore: 0,
|
||||
error: error.message
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
async checkProfanity(content) {
|
||||
try {
|
||||
const isProfane = this.badWordsFilter.isProfane(content);
|
||||
const cleanedContent = this.badWordsFilter.clean(content);
|
||||
|
||||
// Find actual bad words
|
||||
const words = [];
|
||||
if (isProfane) {
|
||||
const tokens = this.tokenizer.tokenize(content.toLowerCase());
|
||||
for (const token of tokens) {
|
||||
if (this.badWordsFilter.isProfane(token)) {
|
||||
words.push(token);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
detected: isProfane,
|
||||
severity: words.length > 3 ? 0.9 : words.length > 1 ? 0.6 : 0.3,
|
||||
words: words,
|
||||
cleanedContent
|
||||
};
|
||||
} catch (error) {
|
||||
logger.error('Profanity check error:', error);
|
||||
return { detected: false, severity: 0 };
|
||||
}
|
||||
}
|
||||
|
||||
async analyzeSentiment(content) {
|
||||
try {
|
||||
const result = this.sentiment.analyze(content);
|
||||
|
||||
return {
|
||||
score: result.score,
|
||||
comparative: result.comparative,
|
||||
positive: result.positive.length,
|
||||
negative: result.negative.length,
|
||||
neutral: result.tokens.length - result.positive.length - result.negative.length
|
||||
};
|
||||
} catch (error) {
|
||||
logger.error('Sentiment analysis error:', error);
|
||||
return { score: 0, comparative: 0 };
|
||||
}
|
||||
}
|
||||
|
||||
async aiContentModeration(content) {
|
||||
if (!this.openai) {
|
||||
return null;
|
||||
}
|
||||
|
||||
try {
|
||||
const response = await this.openai.moderations.create({
|
||||
input: content
|
||||
});
|
||||
|
||||
const result = response.results[0];
|
||||
const violations = [];
|
||||
let riskScore = 0;
|
||||
|
||||
// Check each category
|
||||
for (const [category, flagged] of Object.entries(result.categories)) {
|
||||
if (flagged) {
|
||||
const score = result.category_scores[category];
|
||||
violations.push({
|
||||
type: 'ai_moderation',
|
||||
category,
|
||||
score
|
||||
});
|
||||
riskScore = Math.max(riskScore, score);
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
approved: violations.length === 0,
|
||||
violations,
|
||||
riskScore,
|
||||
flagged: result.flagged
|
||||
};
|
||||
} catch (error) {
|
||||
logger.error('AI moderation error:', error);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async checkToxicity(content) {
|
||||
if (!this.googleLanguage) {
|
||||
return { detected: false, score: 0 };
|
||||
}
|
||||
|
||||
try {
|
||||
const document = {
|
||||
content,
|
||||
type: 'PLAIN_TEXT'
|
||||
};
|
||||
|
||||
const [result] = await this.googleLanguage.analyzeSentiment({ document });
|
||||
const sentiment = result.documentSentiment;
|
||||
|
||||
// Check for negative sentiment with high magnitude
|
||||
const isNegative = sentiment.score < -0.3;
|
||||
const highMagnitude = sentiment.magnitude > 0.7;
|
||||
const isToxic = isNegative && highMagnitude;
|
||||
|
||||
const categories = [];
|
||||
if (isToxic) {
|
||||
if (sentiment.score < -0.7) categories.push('severe_toxicity');
|
||||
else if (sentiment.score < -0.5) categories.push('toxicity');
|
||||
else categories.push('negativity');
|
||||
}
|
||||
|
||||
return {
|
||||
detected: isToxic,
|
||||
score: isToxic ? Math.abs(sentiment.score) : 0,
|
||||
sentiment: {
|
||||
score: sentiment.score,
|
||||
magnitude: sentiment.magnitude
|
||||
},
|
||||
categories
|
||||
};
|
||||
} catch (error) {
|
||||
logger.error('Toxicity check error:', error);
|
||||
return { detected: false, score: 0 };
|
||||
}
|
||||
}
|
||||
|
||||
async moderateBulkContent(contents) {
|
||||
const results = await Promise.all(
|
||||
contents.map(content => this.moderateContent(content))
|
||||
);
|
||||
|
||||
return {
|
||||
total: contents.length,
|
||||
approved: results.filter(r => r.approved).length,
|
||||
rejected: results.filter(r => !r.approved).length,
|
||||
results
|
||||
};
|
||||
}
|
||||
|
||||
async checkContentPolicy(content, policyType = 'default') {
|
||||
const moderation = await this.moderateContent(content);
|
||||
const policy = await this.getContentPolicy(policyType);
|
||||
|
||||
const violations = [];
|
||||
|
||||
// Check against policy thresholds
|
||||
if (moderation.riskScore > policy.maxRiskScore) {
|
||||
violations.push({
|
||||
rule: 'risk_score',
|
||||
threshold: policy.maxRiskScore,
|
||||
actual: moderation.riskScore
|
||||
});
|
||||
}
|
||||
|
||||
if (moderation.sentiment.comparative < policy.minSentiment) {
|
||||
violations.push({
|
||||
rule: 'sentiment',
|
||||
threshold: policy.minSentiment,
|
||||
actual: moderation.sentiment.comparative
|
||||
});
|
||||
}
|
||||
|
||||
return {
|
||||
approved: violations.length === 0,
|
||||
violations,
|
||||
moderation,
|
||||
policy: policyType
|
||||
};
|
||||
}
|
||||
|
||||
async getContentPolicy(type) {
|
||||
const policies = {
|
||||
default: {
|
||||
maxRiskScore: 0.7,
|
||||
minSentiment: -0.5,
|
||||
allowedCategories: []
|
||||
},
|
||||
strict: {
|
||||
maxRiskScore: 0.3,
|
||||
minSentiment: -0.2,
|
||||
allowedCategories: []
|
||||
},
|
||||
permissive: {
|
||||
maxRiskScore: 0.9,
|
||||
minSentiment: -0.8,
|
||||
allowedCategories: ['mild_negativity']
|
||||
}
|
||||
};
|
||||
|
||||
return policies[type] || policies.default;
|
||||
}
|
||||
|
||||
async logViolation(content, result) {
|
||||
try {
|
||||
await ContentViolation.create({
|
||||
content: content.substring(0, 1000), // Limit stored content
|
||||
violations: result.violations,
|
||||
riskScore: result.riskScore,
|
||||
sentiment: result.sentiment,
|
||||
timestamp: new Date()
|
||||
});
|
||||
} catch (error) {
|
||||
logger.error('Failed to log violation:', error);
|
||||
}
|
||||
}
|
||||
|
||||
hashContent(content) {
|
||||
const crypto = require('crypto');
|
||||
return crypto.createHash('sha256').update(content).digest('hex');
|
||||
}
|
||||
|
||||
async getViolationStats(period = '24h') {
|
||||
const since = new Date();
|
||||
|
||||
switch (period) {
|
||||
case '1h':
|
||||
since.setHours(since.getHours() - 1);
|
||||
break;
|
||||
case '24h':
|
||||
since.setDate(since.getDate() - 1);
|
||||
break;
|
||||
case '7d':
|
||||
since.setDate(since.getDate() - 7);
|
||||
break;
|
||||
case '30d':
|
||||
since.setDate(since.getDate() - 30);
|
||||
break;
|
||||
}
|
||||
|
||||
const stats = await ContentViolation.aggregate([
|
||||
{ $match: { timestamp: { $gte: since } } },
|
||||
{
|
||||
$group: {
|
||||
_id: null,
|
||||
total: { $sum: 1 },
|
||||
avgRiskScore: { $avg: '$riskScore' },
|
||||
byType: {
|
||||
$push: {
|
||||
type: { $arrayElemAt: ['$violations.type', 0] }
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
]);
|
||||
|
||||
return stats[0] || { total: 0, avgRiskScore: 0, byType: [] };
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,380 @@
|
||||
import { RateLimiterRedis, RateLimiterMemory } from 'rate-limiter-flexible';
|
||||
import { logger } from '../utils/logger.js';
|
||||
import { RedisClient } from '../config/redis.js';
|
||||
import { RateLimitViolation } from '../models/RateLimitViolation.js';
|
||||
|
||||
export class RateLimiterService {
|
||||
constructor() {
|
||||
this.limiters = new Map();
|
||||
this.redis = null;
|
||||
}
|
||||
|
||||
static getInstance() {
|
||||
if (!RateLimiterService.instance) {
|
||||
RateLimiterService.instance = new RateLimiterService();
|
||||
RateLimiterService.instance.initialize();
|
||||
}
|
||||
return RateLimiterService.instance;
|
||||
}
|
||||
|
||||
async initialize() {
|
||||
this.redis = RedisClient.getInstance();
|
||||
|
||||
// Initialize rate limiters
|
||||
await this.setupRateLimiters();
|
||||
|
||||
logger.info('Rate limiter service initialized');
|
||||
}
|
||||
|
||||
async setupRateLimiters() {
|
||||
const redisClient = this.redis.client;
|
||||
|
||||
// Message sending rate limits
|
||||
this.limiters.set('message:minute', new RateLimiterRedis({
|
||||
storeClient: redisClient,
|
||||
keyPrefix: 'rl:msg:min',
|
||||
points: 30, // 30 messages
|
||||
duration: 60, // per minute
|
||||
blockDuration: 60 // block for 1 minute
|
||||
}));
|
||||
|
||||
this.limiters.set('message:hour', new RateLimiterRedis({
|
||||
storeClient: redisClient,
|
||||
keyPrefix: 'rl:msg:hour',
|
||||
points: 500, // 500 messages
|
||||
duration: 3600, // per hour
|
||||
blockDuration: 600 // block for 10 minutes
|
||||
}));
|
||||
|
||||
this.limiters.set('message:day', new RateLimiterRedis({
|
||||
storeClient: redisClient,
|
||||
keyPrefix: 'rl:msg:day',
|
||||
points: 5000, // 5000 messages
|
||||
duration: 86400, // per day
|
||||
blockDuration: 3600 // block for 1 hour
|
||||
}));
|
||||
|
||||
// Group operations rate limits
|
||||
this.limiters.set('group:join:hour', new RateLimiterRedis({
|
||||
storeClient: redisClient,
|
||||
keyPrefix: 'rl:grp:join',
|
||||
points: 10, // 10 group joins
|
||||
duration: 3600, // per hour
|
||||
blockDuration: 1800 // block for 30 minutes
|
||||
}));
|
||||
|
||||
this.limiters.set('group:create:day', new RateLimiterRedis({
|
||||
storeClient: redisClient,
|
||||
keyPrefix: 'rl:grp:create',
|
||||
points: 5, // 5 group creations
|
||||
duration: 86400, // per day
|
||||
blockDuration: 7200 // block for 2 hours
|
||||
}));
|
||||
|
||||
// API request rate limits
|
||||
this.limiters.set('api:request:second', new RateLimiterRedis({
|
||||
storeClient: redisClient,
|
||||
keyPrefix: 'rl:api:sec',
|
||||
points: 10, // 10 requests
|
||||
duration: 1, // per second
|
||||
blockDuration: 5 // block for 5 seconds
|
||||
}));
|
||||
|
||||
this.limiters.set('api:request:minute', new RateLimiterRedis({
|
||||
storeClient: redisClient,
|
||||
keyPrefix: 'rl:api:min',
|
||||
points: 100, // 100 requests
|
||||
duration: 60, // per minute
|
||||
blockDuration: 60 // block for 1 minute
|
||||
}));
|
||||
|
||||
// Content moderation rate limits
|
||||
this.limiters.set('moderation:minute', new RateLimiterRedis({
|
||||
storeClient: redisClient,
|
||||
keyPrefix: 'rl:mod:min',
|
||||
points: 100, // 100 moderation requests
|
||||
duration: 60, // per minute
|
||||
blockDuration: 30 // block for 30 seconds
|
||||
}));
|
||||
|
||||
// Campaign rate limits
|
||||
this.limiters.set('campaign:hour', new RateLimiterRedis({
|
||||
storeClient: redisClient,
|
||||
keyPrefix: 'rl:camp:hour',
|
||||
points: 5, // 5 campaigns
|
||||
duration: 3600, // per hour
|
||||
blockDuration: 1800 // block for 30 minutes
|
||||
}));
|
||||
|
||||
// Custom rate limiters based on account tier
|
||||
this.setupTierBasedLimiters();
|
||||
}
|
||||
|
||||
setupTierBasedLimiters() {
|
||||
const tiers = {
|
||||
free: {
|
||||
'message:day': 100,
|
||||
'group:join:day': 5,
|
||||
'api:request:day': 1000
|
||||
},
|
||||
basic: {
|
||||
'message:day': 1000,
|
||||
'group:join:day': 20,
|
||||
'api:request:day': 10000
|
||||
},
|
||||
pro: {
|
||||
'message:day': 10000,
|
||||
'group:join:day': 100,
|
||||
'api:request:day': 100000
|
||||
},
|
||||
enterprise: {
|
||||
'message:day': 100000,
|
||||
'group:join:day': 1000,
|
||||
'api:request:day': 1000000
|
||||
}
|
||||
};
|
||||
|
||||
for (const [tier, limits] of Object.entries(tiers)) {
|
||||
for (const [limitType, points] of Object.entries(limits)) {
|
||||
const key = `${tier}:${limitType}`;
|
||||
this.limiters.set(key, new RateLimiterRedis({
|
||||
storeClient: this.redis.client,
|
||||
keyPrefix: `rl:tier:${tier}:${limitType.replace(':', '_')}`,
|
||||
points,
|
||||
duration: 86400, // per day
|
||||
blockDuration: 3600 // block for 1 hour
|
||||
}));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async checkRateLimit(limitType, identifier, options = {}) {
|
||||
const { tier = null, cost = 1 } = options;
|
||||
|
||||
try {
|
||||
// Check tier-specific limit first if tier is provided
|
||||
if (tier) {
|
||||
const tierLimitKey = `${tier}:${limitType}`;
|
||||
if (this.limiters.has(tierLimitKey)) {
|
||||
try {
|
||||
await this.limiters.get(tierLimitKey).consume(identifier, cost);
|
||||
} catch (tierError) {
|
||||
await this.logViolation({
|
||||
limitType: tierLimitKey,
|
||||
identifier,
|
||||
tier,
|
||||
cost
|
||||
});
|
||||
throw tierError;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Check general limit
|
||||
const limiter = this.limiters.get(limitType);
|
||||
if (!limiter) {
|
||||
logger.warn(`Rate limiter not found: ${limitType}`);
|
||||
return { allowed: true };
|
||||
}
|
||||
|
||||
const result = await limiter.consume(identifier, cost);
|
||||
|
||||
return {
|
||||
allowed: true,
|
||||
remainingPoints: result.remainingPoints,
|
||||
msBeforeNext: result.msBeforeNext,
|
||||
consumedPoints: result.consumedPoints,
|
||||
isFirstInDuration: result.isFirstInDuration
|
||||
};
|
||||
} catch (error) {
|
||||
if (error.remainingPoints !== undefined) {
|
||||
// Rate limit exceeded
|
||||
await this.logViolation({
|
||||
limitType,
|
||||
identifier,
|
||||
tier,
|
||||
cost,
|
||||
msBeforeNext: error.msBeforeNext
|
||||
});
|
||||
|
||||
return {
|
||||
allowed: false,
|
||||
remainingPoints: error.remainingPoints,
|
||||
msBeforeNext: error.msBeforeNext,
|
||||
retryAfter: new Date(Date.now() + error.msBeforeNext)
|
||||
};
|
||||
}
|
||||
|
||||
// Actual error
|
||||
logger.error('Rate limit check error:', error);
|
||||
// Fail open on error
|
||||
return { allowed: true, error: error.message };
|
||||
}
|
||||
}
|
||||
|
||||
async checkMultipleRateLimits(limits, identifier, options = {}) {
|
||||
const results = [];
|
||||
|
||||
for (const limitType of limits) {
|
||||
const result = await this.checkRateLimit(limitType, identifier, options);
|
||||
results.push({ limitType, ...result });
|
||||
|
||||
if (!result.allowed) {
|
||||
return {
|
||||
allowed: false,
|
||||
failedLimit: limitType,
|
||||
results
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
allowed: true,
|
||||
results
|
||||
};
|
||||
}
|
||||
|
||||
async getRateLimitStatus(limitTypes, identifier, tier = null) {
|
||||
const status = {};
|
||||
|
||||
for (const limitType of limitTypes) {
|
||||
try {
|
||||
// Check tier-specific limit
|
||||
if (tier) {
|
||||
const tierLimitKey = `${tier}:${limitType}`;
|
||||
if (this.limiters.has(tierLimitKey)) {
|
||||
const tierLimiter = this.limiters.get(tierLimitKey);
|
||||
const tierStatus = await tierLimiter.get(identifier);
|
||||
status[tierLimitKey] = {
|
||||
consumedPoints: tierStatus ? tierStatus.consumedPoints : 0,
|
||||
remainingPoints: tierStatus ? tierStatus.remainingPoints : tierLimiter.points,
|
||||
msBeforeNext: tierStatus ? tierStatus.msBeforeNext : 0
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// Check general limit
|
||||
const limiter = this.limiters.get(limitType);
|
||||
if (limiter) {
|
||||
const limitStatus = await limiter.get(identifier);
|
||||
status[limitType] = {
|
||||
consumedPoints: limitStatus ? limitStatus.consumedPoints : 0,
|
||||
remainingPoints: limitStatus ? limitStatus.remainingPoints : limiter.points,
|
||||
msBeforeNext: limitStatus ? limitStatus.msBeforeNext : 0
|
||||
};
|
||||
}
|
||||
} catch (error) {
|
||||
logger.error(`Failed to get rate limit status for ${limitType}:`, error);
|
||||
status[limitType] = { error: error.message };
|
||||
}
|
||||
}
|
||||
|
||||
return status;
|
||||
}
|
||||
|
||||
async resetRateLimit(limitType, identifier) {
|
||||
try {
|
||||
const limiter = this.limiters.get(limitType);
|
||||
if (!limiter) {
|
||||
throw new Error(`Rate limiter not found: ${limitType}`);
|
||||
}
|
||||
|
||||
await limiter.delete(identifier);
|
||||
logger.info(`Reset rate limit ${limitType} for ${identifier}`);
|
||||
|
||||
return { success: true };
|
||||
} catch (error) {
|
||||
logger.error('Failed to reset rate limit:', error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async setCustomRateLimit(key, config) {
|
||||
try {
|
||||
const limiter = new RateLimiterRedis({
|
||||
storeClient: this.redis.client,
|
||||
keyPrefix: `rl:custom:${key}`,
|
||||
points: config.points,
|
||||
duration: config.duration,
|
||||
blockDuration: config.blockDuration || config.duration
|
||||
});
|
||||
|
||||
this.limiters.set(`custom:${key}`, limiter);
|
||||
logger.info(`Created custom rate limit: ${key}`);
|
||||
|
||||
return { success: true, key: `custom:${key}` };
|
||||
} catch (error) {
|
||||
logger.error('Failed to create custom rate limit:', error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async logViolation(violation) {
|
||||
try {
|
||||
await RateLimitViolation.create({
|
||||
limitType: violation.limitType,
|
||||
identifier: violation.identifier,
|
||||
tier: violation.tier,
|
||||
cost: violation.cost,
|
||||
msBeforeNext: violation.msBeforeNext,
|
||||
timestamp: new Date()
|
||||
});
|
||||
} catch (error) {
|
||||
logger.error('Failed to log rate limit violation:', error);
|
||||
}
|
||||
}
|
||||
|
||||
async getViolationReport(period = '24h') {
|
||||
const since = new Date();
|
||||
|
||||
switch (period) {
|
||||
case '1h':
|
||||
since.setHours(since.getHours() - 1);
|
||||
break;
|
||||
case '24h':
|
||||
since.setDate(since.getDate() - 1);
|
||||
break;
|
||||
case '7d':
|
||||
since.setDate(since.getDate() - 7);
|
||||
break;
|
||||
}
|
||||
|
||||
const report = await RateLimitViolation.aggregate([
|
||||
{ $match: { timestamp: { $gte: since } } },
|
||||
{
|
||||
$group: {
|
||||
_id: {
|
||||
limitType: '$limitType',
|
||||
identifier: '$identifier'
|
||||
},
|
||||
count: { $sum: 1 },
|
||||
totalCost: { $sum: '$cost' },
|
||||
lastViolation: { $max: '$timestamp' }
|
||||
}
|
||||
},
|
||||
{
|
||||
$sort: { count: -1 }
|
||||
},
|
||||
{
|
||||
$limit: 100
|
||||
}
|
||||
]);
|
||||
|
||||
return {
|
||||
period,
|
||||
since,
|
||||
topViolators: report
|
||||
};
|
||||
}
|
||||
|
||||
// Utility method to calculate optimal rate limit settings
|
||||
calculateOptimalLimits(usage, targetSuccessRate = 0.95) {
|
||||
const buffer = 1 / (1 - targetSuccessRate);
|
||||
|
||||
return {
|
||||
points: Math.ceil(usage.peak * buffer),
|
||||
duration: usage.duration,
|
||||
blockDuration: Math.min(usage.duration, 3600)
|
||||
};
|
||||
}
|
||||
}
|
||||
93
marketing-agent/services/safety-guard/src/utils/logger.js
Normal file
93
marketing-agent/services/safety-guard/src/utils/logger.js
Normal file
@@ -0,0 +1,93 @@
|
||||
import winston from 'winston';
|
||||
import path from 'path';
|
||||
import { fileURLToPath } from 'url';
|
||||
|
||||
const __filename = fileURLToPath(import.meta.url);
|
||||
const __dirname = path.dirname(__filename);
|
||||
|
||||
const { combine, timestamp, printf, colorize, errors } = winston.format;
|
||||
|
||||
// Custom log format
|
||||
const logFormat = printf(({ level, message, timestamp, stack, ...metadata }) => {
|
||||
let msg = `${timestamp} [${level}] ${message}`;
|
||||
|
||||
if (stack) {
|
||||
msg += `\n${stack}`;
|
||||
}
|
||||
|
||||
if (Object.keys(metadata).length > 0) {
|
||||
msg += ` ${JSON.stringify(metadata)}`;
|
||||
}
|
||||
|
||||
return msg;
|
||||
});
|
||||
|
||||
// Create logger instance
|
||||
export const logger = winston.createLogger({
|
||||
level: process.env.LOG_LEVEL || 'info',
|
||||
format: combine(
|
||||
errors({ stack: true }),
|
||||
timestamp({ format: 'YYYY-MM-DD HH:mm:ss' }),
|
||||
logFormat
|
||||
),
|
||||
transports: [
|
||||
// Console transport
|
||||
new winston.transports.Console({
|
||||
format: combine(
|
||||
colorize(),
|
||||
logFormat
|
||||
)
|
||||
}),
|
||||
// File transport for errors
|
||||
new winston.transports.File({
|
||||
filename: path.join(__dirname, '../../logs/error.log'),
|
||||
level: 'error',
|
||||
maxsize: 10485760, // 10MB
|
||||
maxFiles: 5
|
||||
}),
|
||||
// File transport for all logs
|
||||
new winston.transports.File({
|
||||
filename: path.join(__dirname, '../../logs/combined.log'),
|
||||
maxsize: 10485760, // 10MB
|
||||
maxFiles: 10
|
||||
})
|
||||
],
|
||||
exceptionHandlers: [
|
||||
new winston.transports.File({
|
||||
filename: path.join(__dirname, '../../logs/exceptions.log')
|
||||
})
|
||||
],
|
||||
rejectionHandlers: [
|
||||
new winston.transports.File({
|
||||
filename: path.join(__dirname, '../../logs/rejections.log')
|
||||
})
|
||||
]
|
||||
});
|
||||
|
||||
// Add request logging helper
|
||||
export const logRequest = (request) => {
|
||||
const { method, path, payload, query, params } = request;
|
||||
logger.info('API Request', {
|
||||
method,
|
||||
path,
|
||||
query,
|
||||
params,
|
||||
payload: payload ? Object.keys(payload) : null
|
||||
});
|
||||
};
|
||||
|
||||
// Add response logging helper
|
||||
export const logResponse = (request, response, processingTime) => {
|
||||
const { method, path } = request;
|
||||
const statusCode = response.statusCode;
|
||||
|
||||
const level = statusCode >= 500 ? 'error' :
|
||||
statusCode >= 400 ? 'warn' : 'info';
|
||||
|
||||
logger[level]('API Response', {
|
||||
method,
|
||||
path,
|
||||
statusCode,
|
||||
processingTime: `${processingTime}ms`
|
||||
});
|
||||
};
|
||||
Reference in New Issue
Block a user