Skip to content

Commit

Permalink
fix: cn and gov endpoint support (#244)
Browse files Browse the repository at this point in the history
  • Loading branch information
@crystall-bitquill
crystall-bitquill authored Oct 31, 2024
1 parent 8e62769 commit fdb4c1c
Show file tree
Hide file tree
Showing 6 changed files with 385 additions and 136 deletions.
22 changes: 11 additions & 11 deletions common/lib/host_list_provider/rds_host_list_provider.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@ export class RdsHostListProvider implements DynamicHostListProvider {
this.initialHost = this.initialHostList[0];
this.hostListProviderService.setInitialConnectionHostInfo(this.initialHost);
this.refreshRateNano = WrapperProperties.CLUSTER_TOPOLOGY_REFRESH_RATE_MS.get(this.properties) * 1000000;
this.rdsUrlType = this.rdsHelper.identifyRdsType(this.originalUrl);
this.rdsUrlType = this.rdsHelper.identifyRdsType(this.initialHost.host);
}

init(): void {
Expand All @@ -104,12 +104,12 @@ export class RdsHostListProvider implements DynamicHostListProvider {
// identification
this.clusterId = this.initialHost.url;
} else if (this.rdsUrlType.isRds) {
const clusterSuggestedResult: ClusterSuggestedResult | null = this.getSuggestedClusterId(this.initialHost.url);
const clusterSuggestedResult: ClusterSuggestedResult | null = this.getSuggestedClusterId(this.initialHost.host);
if (clusterSuggestedResult && clusterSuggestedResult.clusterId) {
this.clusterId = clusterSuggestedResult.clusterId;
this.isPrimaryClusterId = clusterSuggestedResult.isPrimaryClusterId;
} else {
const clusterRdsHostUrl: string | null = this.rdsHelper.getRdsClusterHostUrl(this.initialHost.url);
const clusterRdsHostUrl: string | null = this.rdsHelper.getRdsClusterHostUrl(this.initialHost.host);
if (clusterRdsHostUrl) {
this.clusterId = this.clusterInstanceTemplate.isPortSpecified()
? `${clusterRdsHostUrl}:${this.clusterInstanceTemplate.port}`
Expand Down Expand Up @@ -143,7 +143,7 @@ export class RdsHostListProvider implements DynamicHostListProvider {
}

if (client.targetClient) {
return dialect.getHostRole(client.targetClient, this.properties);
return dialect.getHostRole(client.targetClient);
} else {
throw new AwsWrapperError(Messages.get("AwsClient targetClient not defined."));
}
Expand All @@ -153,7 +153,7 @@ export class RdsHostListProvider implements DynamicHostListProvider {
if (!this.isTopologyAwareDatabaseDialect(dialect)) {
throw new TypeError(Messages.get("RdsHostListProvider.incorrectDialect"));
}
const instanceName = await dialect.identifyConnection(targetClient, this.properties);
const instanceName = await dialect.identifyConnection(targetClient);

return this.refresh(targetClient).then((topology) => {
const matches = topology.filter((host) => host.hostId === instanceName);
Expand Down Expand Up @@ -215,17 +215,17 @@ export class RdsHostListProvider implements DynamicHostListProvider {
}
}

private getSuggestedClusterId(url: string): ClusterSuggestedResult | null {
private getSuggestedClusterId(host: string): ClusterSuggestedResult | null {
for (const [key, hosts] of RdsHostListProvider.topologyCache.getEntries()) {
const isPrimaryCluster: boolean = RdsHostListProvider.primaryClusterIdCache.get(key, false, this.suggestedClusterIdRefreshRateNano) ?? false;
if (key === url) {
return new ClusterSuggestedResult(url, isPrimaryCluster);
if (key === host) {
return new ClusterSuggestedResult(host, isPrimaryCluster);
}

if (hosts) {
for (const host of hosts) {
if (host.url === url) {
logger.debug(Messages.get("RdsHostListProvider.suggestedClusterId", key, url));
for (const hostInfo of hosts) {
if (hostInfo.host === host) {
logger.debug(Messages.get("RdsHostListProvider.suggestedClusterId", key, host));
return new ClusterSuggestedResult(key, isPrimaryCluster);
}
}
Expand Down
1 change: 0 additions & 1 deletion common/lib/utils/connection_url_parser.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,6 @@ export abstract class ConnectionUrlParser {

private getHostInfo(host: string, port: string | undefined, role: HostRole, builder: HostInfoBuilder): HostInfo {
const hostId = ConnectionUrlParser.rdsUtils.getRdsInstanceId(host);

builder = builder.withHost(host).withRole(role);

if (hostId) {
Expand Down
233 changes: 166 additions & 67 deletions common/lib/utils/rds_utils.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
*/

import { RdsUrlType } from "./rds_url_type";
import { equalsIgnoreCase } from "./utils";

export class RdsUtils {
// Aurora DB clusters support different endpoints. More details about Aurora RDS endpoints
Expand Down Expand Up @@ -53,27 +54,49 @@ export class RdsUtils {
//
// Instance Endpoint: <instance-name>.<xyz>.rds.<aws-region>.amazonaws.com.cn
// Example: test-postgres-instance-1.123456789012.rds.cn-northwest-1.amazonaws.com.cn
//
//
// Governmental endpoints
// https://aws.amazon.com/compliance/fips/#FIPS_Endpoints_by_Service
// https://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/s3/model/Region.html

private static readonly AURORA_DNS_PATTERN =
/(?<instance>.+)\.(?<dns>proxy-|cluster-|cluster-ro-|cluster-custom-)?(?<domain>[a-zA-Z0-9]+\.(?<region>[a-zA-Z0-9-]+)\.rds\.amazonaws\.com)/i;
private static readonly AURORA_INSTANCE_PATTERN = /(?<instance>.+)\.(?<domain>[a-zA-Z0-9]+\.(?<region>[a-zA-Z0-9-]+)\.rds\.amazonaws\.com)/i;
/^(?<instance>.+)\.(?<dns>proxy-|cluster-|cluster-ro-|cluster-custom-)?(?<domain>[a-zA-Z0-9]+\.(?<region>[a-zA-Z0-9-]+)\.rds\.amazonaws\.com)$/i;
private static readonly AURORA_INSTANCE_PATTERN = /^(?<instance>.+)\.(?<domain>[a-zA-Z0-9]+\.(?<region>[a-zA-Z0-9-]+)\.rds\.amazonaws\.com)$/i;
private static readonly AURORA_CLUSTER_PATTERN =
/(?<instance>.+)\.(?<dns>cluster-|cluster-ro-)+(?<domain>[a-zA-Z0-9]+\.(?<region>[a-zA-Z0-9-]+)\.rds\.amazonaws\.com)/i;
/^(?<instance>.+)\.(?<dns>cluster-|cluster-ro-)+(?<domain>[a-zA-Z0-9]+\.(?<region>[a-zA-Z0-9-]+)\.rds\.amazonaws\.com)$/i;
private static readonly AURORA_CUSTOM_CLUSTER_PATTERN =
/(?<instance>.+)\.(?<dns>cluster-custom-)+(?<domain>[a-zA-Z0-9]+\.(?<region>[a-zA-Z0-9-]+)\.rds\.amazonaws\.com)/i;
/^(?<instance>.+)\.(?<dns>cluster-custom-)+(?<domain>[a-zA-Z0-9]+\.(?<region>[a-zA-Z0-9-]+)\.rds\.amazonaws\.com)$/i;
private static readonly AURORA_PROXY_DNS_PATTERN =
/(?<instance>.+)\.(?<dns>proxy-)+(?<domain>[a-zA-Z0-9]+\.(?<region>[a-zA-Z0-9-]+)\.rds\.amazonaws\.com)/i;
/^(?<instance>.+)\.(?<dns>proxy-)+(?<domain>[a-zA-Z0-9]+\.(?<region>[a-zA-Z0-9-]+)\.rds\.amazonaws\.com)$/i;
private static readonly AURORA_CHINA_DNS_PATTERN =
/(?<instance>.+)\.(?<dns>proxy-|cluster-|cluster-ro-|cluster-custom-)?(?<domain>[a-zA-Z0-9]+\.rds\.(?<region>[a-zA-Z0-9-]+)\.amazonaws\.com\.cn)/i;
/^(?<instance>.+)\.(?<dns>proxy-|cluster-|cluster-ro-|cluster-custom-)?(?<domain>[a-zA-Z0-9]+\.rds\.(?<region>[a-zA-Z0-9-]+)\.amazonaws\.com\.cn)$/i;
private static readonly AURORA_OLD_CHINA_DNS_PATTERN =
/^(?<instance>.+)\.(?<dns>proxy-|cluster-|cluster-ro-|cluster-custom-)?(?<domain>[a-zA-Z0-9]+\.(?<region>[a-zA-Z0-9-]+)\.rds\.amazonaws\.com\.cn)$/i;
private static readonly AURORA_CHINA_INSTANCE_PATTERN =
/(?<instance>.+)\.(?<domain>[a-zA-Z0-9]+\.(?<region>[a-zA-Z0-9-]+)\.rds\.amazonaws\.com\.cn)/i;
/^(?<instance>.+)\.(?<domain>[a-zA-Z0-9]+\.rds\.(?<region>[a-zA-Z0-9-]+)\.amazonaws\.com\.cn)$/i;
private static readonly AURORA_OLD_CHINA_INSTANCE_PATTERN =
/^(?<instance>.+)\.(?<domain>[a-zA-Z0-9]+\.(?<region>[a-zA-Z0-9-]+)\.rds\.amazonaws\.com\.cn)$/i;
private static readonly AURORA_CHINA_CLUSTER_PATTERN =
/(?<instance>.+)\.(?<dns>cluster-|cluster-ro-)+(?<domain>[a-zA-Z0-9]+\.rds\.(?<region>[a-zA-Z0-9-]+)\.amazonaws\.com\.cn)/i;
/^(?<instance>.+)\.(?<dns>cluster-|cluster-ro-)+(?<domain>[a-zA-Z0-9]+\.rds\.(?<region>[a-zA-Z0-9-]+)\.amazonaws\.com\.cn)$/i;
private static readonly AURORA_OLD_CHINA_CLUSTER_PATTERN =
/^(?<instance>.+)\.(?<dns>cluster-|cluster-ro-)+(?<domain>[a-zA-Z0-9]+\.(?<region>[a-zA-Z0-9-]+)\.rds\.amazonaws\.com\.cn)$/i;
private static readonly AURORA_CHINA_CUSTOM_CLUSTER_PATTERN =
/(?<instance>.+)\.(?<dns>cluster-custom-)+(?<domain>[a-zA-Z0-9]+\.rds\.(?<region>[a-zA-Z0-9-]+)\.amazonaws\.com\.cn)/i;
/^(?<instance>.+)\.(?<dns>cluster-custom-)+(?<domain>[a-zA-Z0-9]+\.rds\.(?<region>[a-zA-Z0-9-]+)\.amazonaws\.com\.cn)$/i;
private static readonly AURORA_OLD_CHINA_CUSTOM_CLUSTER_PATTERN =
/^(?<instance>.+)\.(?<dns>cluster-custom-)+(?<domain>[a-zA-Z0-9]+\.(?<region>[a-zA-Z0-9-]+)\.rds\.amazonaws\.com\.cn)$/i;
private static readonly AURORA_CHINA_PROXY_DNS_PATTERN =
/(?<instance>.+)\.(?<dns>proxy-)+(?<domain>[a-zA-Z0-9]+\.rds\.(?<region>[a-zA-Z0-9-])+\.amazonaws\.com\.cn)/i;
private static readonly ELB_PATTERN = /(?<instance>.+)\.elb\.((?<region>[a-zA-Z0-9-]+)\.amazonaws\.com)/i;
/^(?<instance>.+)\.(?<dns>proxy-)+(?<domain>[a-zA-Z0-9]+\.rds\.(?<region>[a-zA-Z0-9-])+\.amazonaws\.com\.cn)$/i;
private static readonly AURORA_OLD_CHINA_PROXY_DNS_PATTERN =
/^(?<instance>.+)\.(?<dns>proxy-)+(?<domain>[a-zA-Z0-9]+\.(?<region>[a-zA-Z0-9-])+\.rds\.amazonaws\.com\.cn)$/i;

private static readonly AURORA_GOV_DNS_PATTERN =
/^(?<instance>.+)\.(?<dns>proxy-|cluster-|cluster-ro-|cluster-custom-|shardgrp-)?(?<domain>[a-zA-Z0-9]+\.rds\.(?<region>[a-zA-Z0-9-]+)\.(amazonaws\.com|c2s\.ic\.gov|sc2s\.sgov\.gov))$/i;

private static readonly AURORA_GOV_CLUSTER_PATTERN =
/^(?<instance>.+)\.(?<dns>cluster-|cluster-ro-)+(?<domain>[a-zA-Z0-9]+\.rds\.(?<region>[a-zA-Z0-9-]+)\.(amazonaws\.com|c2s\.ic\.gov|sc2s\.sgov\.gov))$/i;

private static readonly ELB_PATTERN = /^(?<instance>.+)\.elb\.((?<region>[a-zA-Z0-9-]+)\.amazonaws\.com)$/i;
private static readonly IP_V4 =
/^(([1-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){1}(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){2}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$/i;
private static readonly IP_V6 = /^[0-9a-fA-F]{1,4}(:[0-9a-fA-F]{1,4}){7}$/i;
Expand All @@ -85,109 +108,136 @@ export class RdsUtils {
static readonly DOMAIN_GROUP = "domain";
static readonly REGION_GROUP = "region";

public isRdsClusterDns(host: string): boolean | null {
return host.match(RdsUtils.AURORA_CLUSTER_PATTERN) != null || host.match(RdsUtils.AURORA_CHINA_CLUSTER_PATTERN) != null;
private static readonly cachedPatterns = new Map();
private static readonly cachedDnsPatterns = new Map();

public isRdsClusterDns(host: string): boolean {
const dnsGroup = this.getDnsGroup(host);
return equalsIgnoreCase(dnsGroup, "cluster-") || equalsIgnoreCase(dnsGroup, "cluster-ro-");
}

public isRdsCustomClusterDns(host: string): boolean {
return host.match(RdsUtils.AURORA_CUSTOM_CLUSTER_PATTERN) != null || host.match(RdsUtils.AURORA_CHINA_CUSTOM_CLUSTER_PATTERN) != null;
const dnsGroup = this.getDnsGroup(host);
return equalsIgnoreCase(dnsGroup, "cluster-custom-");
}

public isRdsDns(host: string) {
return host.match(RdsUtils.AURORA_DNS_PATTERN) || host.match(RdsUtils.AURORA_CHINA_DNS_PATTERN);
const matcher = this.cacheMatcher(
host,
RdsUtils.AURORA_DNS_PATTERN,
RdsUtils.AURORA_CHINA_DNS_PATTERN,
RdsUtils.AURORA_OLD_CHINA_DNS_PATTERN,
RdsUtils.AURORA_GOV_DNS_PATTERN
);
const group = this.getRegexGroup(matcher, RdsUtils.DNS_GROUP);

if (group) {
RdsUtils.cachedDnsPatterns.set(host, group);
}

return matcher != null;
}

public isRdsInstance(host: string): boolean {
return host.match(RdsUtils.AURORA_INSTANCE_PATTERN) !== null || host.match(RdsUtils.AURORA_CHINA_INSTANCE_PATTERN) !== null;
return !this.getDnsGroup(host) && this.isRdsDns(host);
}

isRdsProxyDns(host: string) {
return host.match(RdsUtils.AURORA_PROXY_DNS_PATTERN) || host.match(RdsUtils.AURORA_CHINA_PROXY_DNS_PATTERN);
}

public isElbUrl(host: string) {
return host.match(RdsUtils.ELB_PATTERN);
const dnsGroup = this.getDnsGroup(host);
return dnsGroup && dnsGroup.startsWith("proxy-");
}

public getRdsInstanceId(host: string) {
if (!host) {
return null;
}

const instanceId = (host.match(RdsUtils.AURORA_INSTANCE_PATTERN) || host.match(RdsUtils.AURORA_CHINA_INSTANCE_PATTERN))?.groups?.[
RdsUtils.INSTANCE_GROUP
];
return instanceId ? instanceId : null;
const matcher = this.cacheMatcher(
host,
RdsUtils.AURORA_DNS_PATTERN,
RdsUtils.AURORA_CHINA_DNS_PATTERN,
RdsUtils.AURORA_OLD_CHINA_DNS_PATTERN,
RdsUtils.AURORA_GOV_DNS_PATTERN
);
if (this.getRegexGroup(matcher, RdsUtils.DNS_GROUP)) {
return this.getRegexGroup(matcher, RdsUtils.INSTANCE_GROUP);
}

return null;
}

public getRdsInstanceHostPattern(host: string): string {
if (host == null) {
if (!host) {
return "?";
}

const matcher = host.match(RdsUtils.AURORA_DNS_PATTERN);
if (matcher !== null && matcher.groups !== undefined) {
return "?." + matcher.groups[RdsUtils.DOMAIN_GROUP];
}
const chinaMatcher = host.match(RdsUtils.AURORA_CHINA_DNS_PATTERN);
if (chinaMatcher !== null && chinaMatcher.groups !== undefined) {
return "?." + chinaMatcher.groups[RdsUtils.DOMAIN_GROUP];
}
return "?";
const matcher = this.cacheMatcher(
host,
RdsUtils.AURORA_DNS_PATTERN,
RdsUtils.AURORA_CHINA_DNS_PATTERN,
RdsUtils.AURORA_OLD_CHINA_DNS_PATTERN,
RdsUtils.AURORA_GOV_DNS_PATTERN
);
const group = this.getRegexGroup(matcher, RdsUtils.DOMAIN_GROUP);
return group ? `?.${group}` : "?";
}

public getRdsRegion(host: string): string | null {
const matcher = host.match(RdsUtils.AURORA_DNS_PATTERN);
if (matcher !== null && matcher.groups !== undefined) {
return matcher.groups[RdsUtils.REGION_GROUP];
if (!host) {
return null;
}
const chinaMatcher = host.match(RdsUtils.AURORA_CHINA_DNS_PATTERN);
if (chinaMatcher !== null && chinaMatcher.groups !== undefined) {
return chinaMatcher.groups[RdsUtils.REGION_GROUP];

const matcher = this.cacheMatcher(
host,
RdsUtils.AURORA_DNS_PATTERN,
RdsUtils.AURORA_CHINA_DNS_PATTERN,
RdsUtils.AURORA_OLD_CHINA_DNS_PATTERN,
RdsUtils.AURORA_GOV_DNS_PATTERN
);

const group = this.getRegexGroup(matcher, RdsUtils.REGION_GROUP);
if (group) {
return group;
}

const elbMatcher = host.match(RdsUtils.ELB_PATTERN);
if (elbMatcher !== null && elbMatcher.groups !== undefined) {
return elbMatcher.groups[RdsUtils.REGION_GROUP];
if (elbMatcher && elbMatcher.length > 0) {
return this.getRegexGroup(elbMatcher, RdsUtils.REGION_GROUP);
}

return null;
}

public isWriterClusterDns(host: string) {
if (host === undefined) {
return false;
}

const matcher = host.match(RdsUtils.AURORA_CLUSTER_PATTERN);
if (matcher !== null && matcher.groups !== undefined) {
return "cluster-" === matcher.groups[RdsUtils.DNS_GROUP];
}
const chinaMatcher = host.match(RdsUtils.AURORA_CHINA_CLUSTER_PATTERN);
if (chinaMatcher !== null && chinaMatcher.groups !== undefined) {
return "cluster-" == chinaMatcher.groups[RdsUtils.DNS_GROUP];
}
return false;
public isWriterClusterDns(host: string): boolean {
const dnsGroup = this.getDnsGroup(host);
return equalsIgnoreCase(dnsGroup, "cluster-");
}

public isReaderClusterDns(host: string): boolean {
const matcher = host.match(RdsUtils.AURORA_CLUSTER_PATTERN);
if (matcher !== null && matcher.groups !== undefined) {
return "cluster-ro-" == matcher.groups[RdsUtils.DNS_GROUP];
}
const chinaMatcher = host.match(RdsUtils.AURORA_CHINA_CLUSTER_PATTERN);
if (chinaMatcher !== null && chinaMatcher.groups !== undefined) {
return "cluster-ro-" == chinaMatcher.groups[RdsUtils.DNS_GROUP];
}
return false;
const dnsGroup = this.getDnsGroup(host);
return equalsIgnoreCase(dnsGroup, "cluster-ro-");
}

public getRdsClusterHostUrl(host: string): string | null {
if (!host) {
return null;
}

const matcher = host.match(RdsUtils.AURORA_CLUSTER_PATTERN);
if (matcher) {
return host.replace(RdsUtils.AURORA_CLUSTER_PATTERN, "$<instance>.cluster-$<domain>");
}
const chinaMatcher = host.match(RdsUtils.AURORA_CHINA_CLUSTER_PATTERN);
if (chinaMatcher) {
return host.replace(RdsUtils.AURORA_CHINA_CLUSTER_PATTERN, "${<instance>.cluster-$<domain>");
return host.replace(RdsUtils.AURORA_CHINA_CLUSTER_PATTERN, "$<instance>.cluster-$<domain>");
}
const oldChinaMatcher = host.match(RdsUtils.AURORA_OLD_CHINA_CLUSTER_PATTERN);
if (oldChinaMatcher) {
return host.replace(RdsUtils.AURORA_OLD_CHINA_CLUSTER_PATTERN, "$<instance>.cluster-$<domain>");
}
const govMatcher = host.match(RdsUtils.AURORA_GOV_CLUSTER_PATTERN);
if (govMatcher) {
return host.replace(RdsUtils.AURORA_GOV_CLUSTER_PATTERN, "$<instance>.cluster-$<domain>");
}
return null;
}
Expand Down Expand Up @@ -259,4 +309,53 @@ export class RdsUtils {
}
return hostAndPort.substring(0, index);
}

private getDnsGroup(host: string): string | null {
if (!host) {
return null;
}

const group = RdsUtils.cachedDnsPatterns.get(host);
if (group) {
return group;
}

const matcher = this.cacheMatcher(
host,
RdsUtils.AURORA_DNS_PATTERN,
RdsUtils.AURORA_CHINA_DNS_PATTERN,
RdsUtils.AURORA_OLD_CHINA_DNS_PATTERN,
RdsUtils.AURORA_GOV_DNS_PATTERN
);
return this.getRegexGroup(matcher, RdsUtils.DNS_GROUP);
}

private getRegexGroup(matcher: RegExpMatchArray, groupName: string): string | null {
if (!matcher) {
return null;
}

return matcher.groups?.[groupName] ?? null;
}

private cacheMatcher(host: string, ...patterns: RegExp[]) {
let matcher = null;
for (const pattern of patterns) {
matcher = RdsUtils.cachedPatterns.get(host);
if (matcher) {
return matcher;
}
matcher = host.match(pattern);
if (matcher && matcher.length > 0) {
RdsUtils.cachedPatterns.set(host, matcher);
return matcher;
}
}
return null;
}

static clearCache() {
RdsUtils.cachedPatterns.clear();
RdsUtils.cachedDnsPatterns.clear();
}
}
4 changes: 4 additions & 0 deletions common/lib/utils/utils.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,3 +72,7 @@ export function logAndThrowError(message: string) {
logger.error(message);
throw new AwsWrapperError(message);
}

export function equalsIgnoreCase(value1: string | null, value2: string | null): boolean {
return value1 != null && value2 != null && value1.localeCompare(value2, undefined, { sensitivity: "accent" }) === 0;
}
Loading

0 comments on commit fdb4c1c

Please sign in to comment.