当前位置：首页 > news >正文

遗留系统微服务改造（三）：监控运维与最佳实践总结

news 2025/10/11 6:15:31

遗留系统改造时，大家往往把注意力都放在架构设计和代码实现上，监控和运维反而容易被忽略。我之前就踩过这个坑：项目技术方案设计得挺好，代码也写得不错，结果上线后出问题了，半天找不到根因在哪，最后只能灰溜溜地回滚。

那次之后我明白了一个道理：没有监控的微服务改造，就像闭着眼睛开车。今天就来分享一下如何搭建监控体系，以及改造过程中的一些实用经验。

1. 监控体系建设

1.1 监控的层次

微服务监控可不是简单看看CPU和内存就行了，得分层次来搭建：

基础设施监控：服务器、网络、存储这些底层资源
应用监控：服务性能、错误率、响应时间
业务监控：订单量、用户活跃度、转化率这些业务数据
用户体验监控：页面加载速度、操作是否流畅

这四个层次缺一不可，每一层都能告诉你不同的故事。

1.2 改造过程监控

改造期间，有几个地方特别需要盯紧：

1.2.1 流量切换监控

// 流量切换监控
@Component
public class TrafficSwitchMonitor {private final MeterRegistry meterRegistry;private final Counter legacyRequestCounter;private final Counter newServiceRequestCounter;private final Timer legacyResponseTime;private final Timer newServiceResponseTime;public TrafficSwitchMonitor(MeterRegistry meterRegistry) {this.meterRegistry = meterRegistry;this.legacyRequestCounter = Counter.builder("requests.legacy").description("Requests to legacy system").register(meterRegistry);this.newServiceRequestCounter = Counter.builder("requests.new").description("Requests to new service").register(meterRegistry);this.legacyResponseTime = Timer.builder("response.time.legacy").description("Legacy system response time").register(meterRegistry);this.newServiceResponseTime = Timer.builder("response.time.new").description("New service response time").register(meterRegistry);}public void recordLegacyRequest(Duration responseTime) {legacyRequestCounter.increment();legacyResponseTime.record(responseTime);}public void recordNewServiceRequest(Duration responseTime) {newServiceRequestCounter.increment();newServiceResponseTime.record(responseTime);}// 计算流量分布public TrafficDistribution getCurrentDistribution() {double legacyCount = legacyRequestCounter.count();double newCount = newServiceRequestCounter.count();double total = legacyCount + newCount;if (total == 0) {return new TrafficDistribution(0, 0);}return new TrafficDistribution(legacyCount / total * 100,newCount / total * 100);}
}

1.2.2 数据一致性监控

// 数据一致性监控
@Component
public class DataConsistencyMonitor {@Autowiredprivate LegacyDataSource legacyDataSource;@Autowiredprivate NewServiceDataSource newDataSource;private final Gauge dataConsistencyGauge;private final Counter inconsistencyCounter;public DataConsistencyMonitor(MeterRegistry meterRegistry) {this.dataConsistencyGauge = Gauge.builder("data.consistency.percentage").description("Data consistency percentage").register(meterRegistry, this, DataConsistencyMonitor::calculateConsistencyPercentage);this.inconsistencyCounter = Counter.builder("data.inconsistency").description("Number of data inconsistencies detected").register(meterRegistry);}@Scheduled(fixedDelay = 300000) // 每5分钟检查一次public void checkDataConsistency() {try {List<Long> sampleIds = getSampleIds(1000);int consistentCount = 0;for (Long id : sampleIds) {if (isDataConsistent(id)) {consistentCount++;} else {inconsistencyCounter.increment(Tags.of("entity", "user", "id", id.toString()));logger.warn("Data inconsistency detected for user: {}", id);}}double consistencyPercentage = (double) consistentCount / sampleIds.size() * 100;logger.info("Data consistency check completed: {:.2f}%", consistencyPercentage);// 如果一致性低于95%，发送告警if (consistencyPercentage < 95.0) {alertService.sendAlert("Low data consistency", String.format("Data consistency: %.2f%%", consistencyPercentage));}} catch (Exception e) {logger.error("Data consistency check failed", e);}}private boolean isDataConsistent(Long userId) {try {User legacyUser = getUserFromLegacy(userId);User newUser = getUserFromNew(userId);return Objects.equals(legacyUser, newUser);} catch (Exception e) {logger.error("Failed to check consistency for user: " + userId, e);return false;}}private double calculateConsistencyPercentage() {// 返回最近的一致性百分比return lastConsistencyPercentage;}
}

1.2.3 性能对比监控

// 性能对比监控
@Component
public class PerformanceComparisonMonitor {private final Timer legacyServiceTimer;private final Timer newServiceTimer;private final Counter legacyErrorCounter;private final Counter newServiceErrorCounter;public PerformanceComparisonMonitor(MeterRegistry meterRegistry) {this.legacyServiceTimer = Timer.builder("service.response.time").tag("service", "legacy").register(meterRegistry);this.newServiceTimer = Timer.builder("service.response.time").tag("service", "new").register(meterRegistry);this.legacyErrorCounter = Counter.builder("service.errors").tag("service", "legacy").register(meterRegistry);this.newServiceErrorCounter = Counter.builder("service.errors").tag("service", "new").register(meterRegistry);}public void recordLegacyServiceCall(Duration duration, boolean isError) {legacyServiceTimer.record(duration);if (isError) {legacyErrorCounter.increment();}}public void recordNewServiceCall(Duration duration, boolean isError) {newServiceTimer.record(duration);if (isError) {newServiceErrorCounter.increment();}}// 生成性能对比报告@Scheduled(fixedDelay = 600000) // 每10分钟生成一次报告public void generatePerformanceReport() {PerformanceReport report = PerformanceReport.builder().legacyAvgResponseTime(legacyServiceTimer.mean(TimeUnit.MILLISECONDS)).newServiceAvgResponseTime(newServiceTimer.mean(TimeUnit.MILLISECONDS)).legacyErrorRate(calculateErrorRate(legacyErrorCounter, legacyServiceTimer)).newServiceErrorRate(calculateErrorRate(newServiceErrorCounter, newServiceTimer)).timestamp(Instant.now()).build();logger.info("Performance comparison report: {}", report);// 如果新服务性能明显差于旧服务，发送告警if (report.getNewServiceAvgResponseTime() > report.getLegacyAvgResponseTime() * 1.5) {alertService.sendAlert("New service performance degradation", String.format("New service response time: %.2fms, Legacy: %.2fms", report.getNewServiceAvgResponseTime(), report.getLegacyAvgResponseTime()));}}private double calculateErrorRate(Counter errorCounter, Timer timer) {double totalRequests = timer.count();double errorCount = errorCounter.count();return totalRequests > 0 ? errorCount / totalRequests * 100 : 0;}
}

1.3 健康检查机制

// 系统健康检查
@Component
public class MigrationHealthIndicator implements HealthIndicator {@Autowiredprivate LegacySystemClient legacyClient;@Autowiredprivate List<MicroserviceClient> microserviceClients;@Autowiredprivate DataSyncService dataSyncService;@Overridepublic Health health() {Health.Builder builder = Health.up();try {// 检查遗留系统状态boolean legacyHealthy = checkLegacySystemHealth();builder.withDetail("legacy-system", legacyHealthy ? "UP" : "DOWN");// 检查各个微服务状态Map<String, String> serviceStatus = checkMicroservicesHealth();builder.withDetails(serviceStatus);// 检查数据同步状态boolean dataSyncHealthy = checkDataSyncHealth();builder.withDetail("data-sync", dataSyncHealthy ? "UP" : "DOWN");// 检查整体系统状态boolean overallHealthy = legacyHealthy && serviceStatus.values().stream().allMatch("UP"::equals) && dataSyncHealthy;return overallHealthy ? builder.build() : builder.down().build();} catch (Exception e) {return Health.down(e).build();}}private boolean checkLegacySystemHealth() {try {return legacyClient.healthCheck();} catch (Exception e) {logger.error("Legacy system health check failed", e);return false;}}private Map<String, String> checkMicroservicesHealth() {Map<String, String> status = new HashMap<>();for (MicroserviceClient client : microserviceClients) {try {boolean healthy = client.healthCheck();status.put(client.getServiceName(), healthy ? "UP" : "DOWN");} catch (Exception e) {logger.error("Health check failed for service: " + client.getServiceName(), e);status.put(client.getServiceName(), "DOWN");}}return status;}private boolean checkDataSyncHealth() {try {// 检查数据同步延迟是否在可接受范围内Duration syncDelay = dataSyncService.getCurrentSyncDelay();return syncDelay.toMinutes() < 5; // 延迟小于5分钟认为正常} catch (Exception e) {logger.error("Data sync health check failed", e);return false;}}
}

2. 自动化运维

2.1 自动回滚机制

// 自动回滚服务
@Component
public class AutoRollbackService {@Autowiredprivate MetricsService metricsService;@Autowiredprivate TrafficRoutingService routingService;@Autowiredprivate AlertService alertService;@Autowiredprivate AuditService auditService;private final RollbackConfig rollbackConfig;public AutoRollbackService(RollbackConfig rollbackConfig) {this.rollbackConfig = rollbackConfig;}@Scheduled(fixedDelay = 30000) // 每30秒检查一次public void checkSystemHealth() {SystemHealthMetrics metrics = metricsService.getCurrentMetrics();// 检查错误率if (metrics.getErrorRate() > rollbackConfig.getMaxErrorRate()) {logger.warn("High error rate detected: {}", metrics.getErrorRate());triggerRollback("High error rate: " + metrics.getErrorRate());return;}// 检查响应时间if (metrics.getAverageResponseTime().toMillis() > rollbackConfig.getMaxResponseTimeMs()) {logger.warn("High response time detected: {}ms", metrics.getAverageResponseTime().toMillis());triggerRollback("High response time: " + metrics.getAverageResponseTime().toMillis() + "ms");return;}// 检查系统可用性if (metrics.getAvailability() < rollbackConfig.getMinAvailability()) {logger.warn("Low availability detected: {}", metrics.getAvailability());triggerRollback("Low availability: " + metrics.getAvailability());return;}// 检查数据一致性if (metrics.getDataConsistency() < rollbackConfig.getMinDataConsistency()) {logger.warn("Low data consistency detected: {}", metrics.getDataConsistency());triggerRollback("Low data consistency: " + metrics.getDataConsistency());}}private void triggerRollback(String reason) {logger.error("Triggering automatic rollback due to: {}", reason);try {// 1. 将流量切回遗留系统routingService.routeToLegacySystem();logger.info("Traffic routed back to legacy system");// 2. 停止数据同步dataSyncService.pauseSync();logger.info("Data sync paused");// 3. 发送紧急告警alertService.sendEmergencyAlert("Automatic rollback triggered", reason);// 4. 记录回滚事件RollbackEvent event = RollbackEvent.builder().reason(reason).timestamp(Instant.now()).triggeredBy("AUTO_ROLLBACK_SERVICE").build();auditService.recordRollback(event);logger.info("Automatic rollback completed successfully");} catch (Exception e) {logger.error("Automatic rollback failed", e);alertService.sendEmergencyAlert("Automatic rollback failed", "Original reason: " + reason + ", Rollback error: " + e.getMessage());}}
}

2.2 配置管理

// 动态配置管理
@Component
@ConfigurationProperties(prefix = "migration")
@RefreshScope
public class MigrationConfig {private TrafficRouting trafficRouting = new TrafficRouting();private DataSync dataSync = new DataSync();private Monitoring monitoring = new Monitoring();private Rollback rollback = new Rollback();@Datapublic static class TrafficRouting {private int newServicePercentage = 10; // 新服务流量百分比private List<String> canaryUsers = new ArrayList<>(); // 金丝雀用户private boolean enableGrayRelease = true; // 是否启用灰度发布}@Datapublic static class DataSync {private boolean enabled = true;private int batchSize = 1000;private long delayMs = 100;private int maxRetries = 3;}@Datapublic static class Monitoring {private long healthCheckIntervalMs = 30000;private long metricsCollectionIntervalMs = 60000;private double alertThreshold = 0.95; // 告警阈值}@Datapublic static class Rollback {private double maxErrorRate = 0.05; // 最大错误率5%private long maxResponseTimeMs = 5000; // 最大响应时间5秒private double minAvailability = 0.99; // 最小可用性99%private double minDataConsistency = 0.95; // 最小数据一致性95%private boolean autoRollbackEnabled = true;}// getters and setters...
}

2.3 部署自动化

# Kubernetes部署配置
apiVersion: apps/v1
kind: Deployment
metadata:name: user-servicelabels:app: user-serviceversion: v2
spec:replicas: 3selector:matchLabels:app: user-serviceversion: v2template:metadata:labels:app: user-serviceversion: v2spec:containers:- name: user-serviceimage: user-service:v2.0.0ports:- containerPort: 8080env:- name: SPRING_PROFILES_ACTIVEvalue: "production"- name: DATABASE_URLvalueFrom:secretKeyRef:name: db-secretkey: urllivenessProbe:httpGet:path: /actuator/healthport: 8080initialDelaySeconds: 30periodSeconds: 10readinessProbe:httpGet:path: /actuator/health/readinessport: 8080initialDelaySeconds: 5periodSeconds: 5resources:requests:memory: "512Mi"cpu: "250m"limits:memory: "1Gi"cpu: "500m"
---
apiVersion: v1
kind: Service
metadata:name: user-service
spec:selector:app: user-serviceports:- port: 80targetPort: 8080type: ClusterIP

# Istio流量管理配置
apiVersion: networking.istio.io/v1alpha3
kind: VirtualService
metadata:name: user-service
spec:http:- match:- headers:canary:exact: "true"route:- destination:host: user-servicesubset: v2weight: 100- route:- destination:host: user-servicesubset: v1weight: 90- destination:host: user-servicesubset: v2weight: 10
---
apiVersion: networking.istio.io/v1alpha3
kind: DestinationRule
metadata:name: user-service
spec:host: user-servicesubsets:- name: v1labels:version: v1- name: v2labels:version: v2

3. 常见问题与解决方案

3.1 数据一致性如何保证？

这是改造过程中最头疼的问题。原来所有数据都在一个库里，现在分散到各个服务，怎么保证数据不乱套？

我的解决思路：

最终一致性：允许短时间内数据不一致，通过消息机制最终达到一致

// 最终一致性实现
@Component
public class EventualConsistencyHandler {@Autowiredprivate MessageQueue messageQueue;@EventListenerpublic void handleUserUpdate(UserUpdateEvent event) {// 发送用户更新事件到所有相关服务UserSyncMessage message = UserSyncMessage.builder().userId(event.getUserId()).operation("UPDATE").timestamp(Instant.now()).data(event.getUserData()).version(event.getVersion()).build();// 发送到订单服务messageQueue.send("order.user.sync", message);// 发送到推荐服务messageQueue.send("recommendation.user.sync", message);// 发送到通知服务messageQueue.send("notification.user.sync", message);}
}

分布式事务：使用Saga模式或TCC模式处理跨服务的事务

// Saga事务协调器
@Component
public class SagaOrchestrator {public void executeOrderSaga(OrderRequest request) {SagaDefinition saga = SagaDefinition.builder().step("validateUser").action(() -> userService.validateUser(request.getUserId())).compensation(() -> userService.releaseUserLock(request.getUserId())).step("reserveInventory").action(() -> inventoryService.reserve(request.getItems())).compensation(() -> inventoryService.release(request.getItems())).step("createOrder").action(() -> orderService.create(request)).compensation(() -> orderService.cancel(request.getOrderId())).step("processPayment").action(() -> paymentService.process(request.getPayment())).compensation(() -> paymentService.refund(request.getPayment())).build();sagaManager.execute(saga);}
}

事件驱动：通过事件来同步各个服务的数据

数据补偿：定期检查数据，发现不一致就修复

3.2 服务间通信如何优化？

服务拆分后，原来的方法调用变成了网络调用，性能和稳定性都是问题。

我的优化策略：

异步消息：能异步的就异步，用RabbitMQ、Kafka这些中间件

// 异步消息处理
@Component
public class AsyncMessageHandler {@RabbitListener(queues = "order.created")public void handleOrderCreated(OrderCreatedEvent event) {// 异步处理订单创建后的业务逻辑try {// 更新库存inventoryService.updateInventory(event.getItems());// 发送通知notificationService.sendOrderConfirmation(event.getOrderId());// 更新用户积分pointsService.addPoints(event.getUserId(), event.getAmount());} catch (Exception e) {logger.error("Failed to handle order created event", e);// 发送到死信队列重试deadLetterService.send(event);}}
}

服务熔断降级：使用Hystrix、Sentinel等工具防止故障传播

// 服务熔断降级
@Component
public class UserServiceClient {@SentinelResource(value = "getUserInfo", fallback = "getUserInfoFallback")public UserInfo getUserInfo(Long userId) {// 调用用户服务return restTemplate.getForObject("/users/" + userId, UserInfo.class);}public UserInfo getUserInfoFallback(Long userId, BlockException ex) {// 降级处理：返回缓存数据或默认数据UserInfo cachedUser = userCacheService.get(userId);if (cachedUser != null) {return cachedUser;}// 返回默认用户信息return UserInfo.builder().id(userId).username("Unknown User").build();}
}

缓存策略：合理使用缓存，减少不必要的服务调用

服务网格：用Istio这样的Service Mesh统一管理通信

3.3 如何处理遗留系统的技术债务？

老系统用了这么多年，技术债务肯定不少。改造的时候怎么处理这些历史包袱？

我的处理方式：

先评估再动手：列个清单，看看哪些债务影响最大，哪些最好解决

// 技术债务评估工具
@Component
public class TechnicalDebtAnalyzer {public TechnicalDebtReport analyzeTechnicalDebt(String codebasePath) {TechnicalDebtReport report = new TechnicalDebtReport();// 1. 代码复杂度分析ComplexityMetrics complexity = analyzeComplexity(codebasePath);report.setComplexityScore(complexity.getScore());// 2. 代码重复度分析DuplicationMetrics duplication = analyzeDuplication(codebasePath);report.setDuplicationScore(duplication.getScore());// 3. 测试覆盖率分析CoverageMetrics coverage = analyzeCoverage(codebasePath);report.setCoverageScore(coverage.getScore());// 4. 依赖分析DependencyMetrics dependencies = analyzeDependencies(codebasePath);report.setDependencyScore(dependencies.getScore());// 5. 生成改进建议List<ImprovementSuggestion> suggestions = generateSuggestions(report);report.setSuggestions(suggestions);return report;}
}