Skip to content

Commit c88a6fe

Browse files
maxhniebergallmaxhniebergall
authored
[ML] Add mixed cluster tests for inference (elastic#108392)
* mixed cluster tests are executable * add tests from upgrade tests * [ML] Add mixed cluster tests for existing services * clean up * review improvements * spotless * remove blocked AzureOpenAI mixed IT * improvements from DK review * temp for testing * refactoring and documentation * Revert manual testing configs of "temp for testing" This reverts parts of commit fca46fd. * revert TESTING.asciidoc formatting * Update TESTING.asciidoc to avoid reformatting * add minimum version for tests to match minimum version in services * spotless
1 parent 74ec90b commit c88a6fe

File tree

8 files changed

+896
-5
lines changed

8 files changed

+896
-5
lines changed

‎TESTING.asciidoc

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -551,13 +551,19 @@ When running `./gradlew check`, minimal bwc checks are also run against compatib
551551

552552
==== BWC Testing against a specific remote/branch
553553

554-
Sometimes a backward compatibility change spans two versions. A common case is a new functionality
555-
that needs a BWC bridge in an unreleased versioned of a release branch (for example, 5.x).
556-
To test the changes, you can instruct Gradle to build the BWC version from another remote/branch combination instead of
557-
pulling the release branch from GitHub. You do so using the `bwc.remote` and `bwc.refspec.BRANCH` system properties:
554+
Sometimes a backward compatibility change spans two versions.
555+
A common case is a new functionality that needs a BWC bridge in an unreleased versioned of a release branch (for example, 5.x).
556+
Another use case, since the introduction of serverless, is to test BWC against main in addition to the other released branches.
557+
To do so, specify the `bwc.refspec` remote and branch to use for the BWC build as `origin/main`.
558+
To test against main, you will also need to create a new version in link:./server/src/main/java/org/elasticsearch/Version.java[Version.java],
559+
increment `elasticsearch` in link:./build-tools-internal/version.properties[version.properties], and hard-code the `project.version` for ml-cpp
560+
in link:./x-pack/plugin/ml/build.gradle[ml/build.gradle].
561+
562+
In general, to test the changes, you can instruct Gradle to build the BWC version from another remote/branch combination instead of pulling the release branch from GitHub.
563+
You do so using the `bwc.refspec.{VERSION}` system property:
558564

559565
-------------------------------------------------
560-
./gradlew check -Dbwc.remote=${remote} -Dbwc.refspec.5.x=index_req_bwc_5.x
566+
./gradlew check -Dtests.bwc.refspec.8.15=origin/main
561567
-------------------------------------------------
562568

563569
The branch needs to be available on the remote that the BWC makes of the

‎x-pack/plugin/inference/qa/mixed-cluster/build.gradle

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
import org.elasticsearch.gradle.Version
2+
import org.elasticsearch.gradle.VersionProperties
3+
import org.elasticsearch.gradle.util.GradleUtils
4+
import org.elasticsearch.gradle.internal.info.BuildParams
5+
import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask
6+
7+
apply plugin: 'elasticsearch.internal-java-rest-test'
8+
apply plugin: 'elasticsearch.internal-test-artifact-base'
9+
apply plugin: 'elasticsearch.bwc-test'
10+
11+
dependencies {
12+
testImplementation project(path: ':x-pack:plugin:inference:qa:inference-service-tests')
13+
compileOnly project(':x-pack:plugin:core')
14+
javaRestTestImplementation(testArtifact(project(xpackModule('core'))))
15+
javaRestTestImplementation project(path: xpackModule('inference'))
16+
clusterPlugins project(
17+
':x-pack:plugin:inference:qa:test-service-plugin'
18+
)
19+
}
20+
21+
// inference is available in 8.11 or later
22+
def supportedVersion = bwcVersion -> {
23+
return bwcVersion.onOrAfter(Version.fromString("8.11.0"));
24+
}
25+
26+
BuildParams.bwcVersions.withWireCompatible(supportedVersion) { bwcVersion, baseName ->
27+
def javaRestTest = tasks.register("v${bwcVersion}#javaRestTest", StandaloneRestIntegTestTask) {
28+
usesBwcDistribution(bwcVersion)
29+
systemProperty("tests.old_cluster_version", bwcVersion)
30+
maxParallelForks = 1
31+
}
32+
33+
tasks.register(bwcTaskName(bwcVersion)) {
34+
dependsOn javaRestTest
35+
}
36+
}
37+

‎x-pack/plugin/inference/qa/mixed-cluster/src/javaRestTest/java/org/elasticsearch/xpack/inference/qa/mixed/BaseMixedTestCase.java

Lines changed: 129 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,129 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0; you may not use this file except in compliance with the Elastic License
5+
* 2.0.
6+
*/
7+
8+
package org.elasticsearch.xpack.inference.qa.mixed;
9+
10+
import org.apache.http.util.EntityUtils;
11+
import org.elasticsearch.client.Request;
12+
import org.elasticsearch.client.Response;
13+
import org.elasticsearch.common.Strings;
14+
import org.elasticsearch.common.settings.SecureString;
15+
import org.elasticsearch.common.settings.Settings;
16+
import org.elasticsearch.common.util.concurrent.ThreadContext;
17+
import org.elasticsearch.inference.TaskType;
18+
import org.elasticsearch.test.ESTestCase;
19+
import org.elasticsearch.test.http.MockWebServer;
20+
import org.elasticsearch.test.rest.ESRestTestCase;
21+
import org.hamcrest.Matchers;
22+
23+
import java.io.IOException;
24+
import java.util.List;
25+
import java.util.Map;
26+
27+
public abstract class BaseMixedTestCase extends MixedClusterSpecTestCase {
28+
protected static String getUrl(MockWebServer webServer) {
29+
return Strings.format("http://%s:%s", webServer.getHostName(), webServer.getPort());
30+
}
31+
32+
@Override
33+
protected Settings restClientSettings() {
34+
String token = ESRestTestCase.basicAuthHeaderValue("x_pack_rest_user", new SecureString("x-pack-test-password".toCharArray()));
35+
return Settings.builder().put(ThreadContext.PREFIX + ".Authorization", token).build();
36+
}
37+
38+
protected void delete(String inferenceId, TaskType taskType) throws IOException {
39+
var request = new Request("DELETE", Strings.format("_inference/%s/%s", taskType, inferenceId));
40+
var response = ESRestTestCase.client().performRequest(request);
41+
ESRestTestCase.assertOK(response);
42+
}
43+
44+
protected void delete(String inferenceId) throws IOException {
45+
var request = new Request("DELETE", Strings.format("_inference/%s", inferenceId));
46+
var response = ESRestTestCase.client().performRequest(request);
47+
ESRestTestCase.assertOK(response);
48+
}
49+
50+
protected Map<String, Object> getAll() throws IOException {
51+
var request = new Request("GET", "_inference/_all");
52+
var response = ESRestTestCase.client().performRequest(request);
53+
ESRestTestCase.assertOK(response);
54+
return ESRestTestCase.entityAsMap(response);
55+
}
56+
57+
protected Map<String, Object> get(String inferenceId) throws IOException {
58+
var endpoint = Strings.format("_inference/%s", inferenceId);
59+
var request = new Request("GET", endpoint);
60+
var response = ESRestTestCase.client().performRequest(request);
61+
ESRestTestCase.assertOK(response);
62+
return ESRestTestCase.entityAsMap(response);
63+
}
64+
65+
protected Map<String, Object> get(TaskType taskType, String inferenceId) throws IOException {
66+
var endpoint = Strings.format("_inference/%s/%s", taskType, inferenceId);
67+
var request = new Request("GET", endpoint);
68+
var response = ESRestTestCase.client().performRequest(request);
69+
ESRestTestCase.assertOK(response);
70+
return ESRestTestCase.entityAsMap(response);
71+
}
72+
73+
protected Map<String, Object> inference(String inferenceId, TaskType taskType, String input) throws IOException {
74+
var endpoint = Strings.format("_inference/%s/%s", taskType, inferenceId);
75+
var request = new Request("POST", endpoint);
76+
request.setJsonEntity("{\"input\": [" + '"' + input + '"' + "]}");
77+
78+
var response = ESRestTestCase.client().performRequest(request);
79+
ESRestTestCase.assertOK(response);
80+
return ESRestTestCase.entityAsMap(response);
81+
}
82+
83+
protected Map<String, Object> rerank(String inferenceId, List<String> inputs, String query) throws IOException {
84+
var endpoint = Strings.format("_inference/rerank/%s", inferenceId);
85+
var request = new Request("POST", endpoint);
86+
87+
StringBuilder body = new StringBuilder("{").append("\"query\":\"").append(query).append("\",").append("\"input\":[");
88+
89+
for (int i = 0; i < inputs.size(); i++) {
90+
body.append("\"").append(inputs.get(i)).append("\"");
91+
if (i < inputs.size() - 1) {
92+
body.append(",");
93+
}
94+
}
95+
96+
body.append("]}");
97+
request.setJsonEntity(body.toString());
98+
99+
var response = ESRestTestCase.client().performRequest(request);
100+
ESRestTestCase.assertOK(response);
101+
return ESRestTestCase.entityAsMap(response);
102+
}
103+
104+
protected void put(String inferenceId, String modelConfig, TaskType taskType) throws IOException {
105+
String endpoint = Strings.format("_inference/%s/%s?error_trace", taskType, inferenceId);
106+
var request = new Request("PUT", endpoint);
107+
request.setJsonEntity(modelConfig);
108+
var response = ESRestTestCase.client().performRequest(request);
109+
logger.warn("PUT response: {}", response.toString());
110+
System.out.println("PUT response: " + response.toString());
111+
ESRestTestCase.assertOKAndConsume(response);
112+
}
113+
114+
protected static void assertOkOrCreated(Response response) throws IOException {
115+
int statusCode = response.getStatusLine().getStatusCode();
116+
// Once EntityUtils.toString(entity) is called the entity cannot be reused.
117+
// Avoid that call with check here.
118+
if (statusCode == 200 || statusCode == 201) {
119+
return;
120+
}
121+
122+
String responseStr = EntityUtils.toString(response.getEntity());
123+
ESTestCase.assertThat(
124+
responseStr,
125+
response.getStatusLine().getStatusCode(),
126+
Matchers.anyOf(Matchers.equalTo(200), Matchers.equalTo(201))
127+
);
128+
}
129+
}

0 commit comments

Comments
 (0)