diff --git a/java/src/main/java/com/google/appengine/tools/mapreduce/MapReduceJob.java b/java/src/main/java/com/google/appengine/tools/mapreduce/MapReduceJob.java index 86c74280..8b56d0f6 100644 --- a/java/src/main/java/com/google/appengine/tools/mapreduce/MapReduceJob.java +++ b/java/src/main/java/com/google/appengine/tools/mapreduce/MapReduceJob.java @@ -204,6 +204,7 @@ public Value> run() { new HashingSharder(getNumOutputFiles(readers.size())), GoogleCloudStorageFileOutput.BaseOptions.builder() .serviceAccountKey(settings.getServiceAccountKey()) + .projectId(settings.getProjectId()) .build() ); output.setContext(context); diff --git a/java/src/main/java/com/google/appengine/tools/mapreduce/impl/sort/SortWorker.java b/java/src/main/java/com/google/appengine/tools/mapreduce/impl/sort/SortWorker.java index cc0b73c3..34992dfe 100644 --- a/java/src/main/java/com/google/appengine/tools/mapreduce/impl/sort/SortWorker.java +++ b/java/src/main/java/com/google/appengine/tools/mapreduce/impl/sort/SortWorker.java @@ -21,7 +21,6 @@ import java.util.ArrayList; import java.util.List; import java.util.concurrent.TimeUnit; -import java.util.logging.Logger; /** * Sorts a set of keyValues by a lexicographical comparison of the bytes of the key. On beginSlice a @@ -132,7 +131,7 @@ private int getStoredSize() { } /** - * Re arranges the pointers so that they are ordered according to the order of the corresponding + * Re-arranges the pointers so that they are ordered according to the order of the corresponding * keys. */ private void sortData() { diff --git a/java/src/test/java/com/google/appengine/tools/mapreduce/CloudStorageIntegrationTestHelper.java b/java/src/test/java/com/google/appengine/tools/mapreduce/CloudStorageIntegrationTestHelper.java deleted file mode 100644 index f41e6082..00000000 --- a/java/src/test/java/com/google/appengine/tools/mapreduce/CloudStorageIntegrationTestHelper.java +++ /dev/null @@ -1,113 +0,0 @@ -package com.google.appengine.tools.mapreduce; - -import com.google.appengine.tools.development.testing.LocalServiceTestConfig; -import com.google.auth.Credentials; -import com.google.auth.oauth2.ServiceAccountCredentials; -import com.google.cloud.storage.BucketInfo; -import com.google.cloud.storage.Storage; -import com.google.cloud.storage.StorageOptions; -import com.google.cloud.storage.testing.RemoteStorageHelper; -import lombok.Getter; -import lombok.SneakyThrows; - -import java.io.ByteArrayInputStream; -import java.time.Duration; -import java.util.Base64; -import java.util.Collections; -import java.util.List; -import java.util.concurrent.TimeUnit; -import java.util.logging.Level; -import java.util.logging.Logger; - -/** - * sets up storage bucket for tests - * - * as of Apr 2020, no gcs emulator https://cloud.google.com/sdk/gcloud/reference/beta/emulators - * - * @see "https://googleapis.dev/java/google-cloud-storage/1.106.0/com/google/cloud/storage/testing/RemoteStorageHelper.html" - * - * eg, - * 1. create SA in target project; give it "Storage Admin" role - * 2. cat ~/Downloads/worklytics-ci-111242f427df.json | base64 - * 3. set output of that as your env variable - * - in IntelliJ, set this via RunConfigurations --> Env Variables. - * - in GitHub, set it as via repo --> Settings --> Secrets so it can be utilized in workflows - * - * q: better to have a helper class for this? analogous to - * @see com.google.appengine.tools.development.testing.LocalServiceTestHelper - */ -public class CloudStorageIntegrationTestHelper implements LocalServiceTestConfig { - - public final String KEY_ENV_VAR = "APPENGINE_MAPREDUCE_CI_SERVICE_ACCOUNT_KEY"; - - @Getter - Storage storage; - @Getter - static String bucket; - @Getter - String projectId; - @Getter - Credentials credentials; - - @Getter - String base64EncodedServiceAccountKey; - - @SneakyThrows - @Override - public void setUp() { - - String keyVar = System.getenv(KEY_ENV_VAR); - - - if (keyVar == null) { - //attempt w default credentials - credentials = StorageOptions.getDefaultInstance().getCredentials(); - - //TODO: more elegant solution? weirdness seems to happen if mix projects; credentials' project - // isn't exposed to java code via any public interface; yet bucket is created in the project - // to which the credentials default project is set - projectId = "worklytics-ci"; - //throw new IllegalStateException("Must set environment variable " + KEY_ENV_VAR + " as base64 encoded service account key to use for storage integration tests"); - } else { - base64EncodedServiceAccountKey = keyVar.trim(); - String jsonKey = new String(Base64.getDecoder().decode(base64EncodedServiceAccountKey.getBytes())); - - credentials = ServiceAccountCredentials.fromStream(new ByteArrayInputStream(jsonKey.getBytes())); - projectId = ((ServiceAccountCredentials) credentials).getProjectId(); - } - - storage = StorageOptions.newBuilder() - .setCredentials(credentials) - .setProjectId(projectId) - .build().getService(); - if (bucket == null) { - bucket = RemoteStorageHelper.generateBucketName(); - - //avoid test data being retained forever, even if bucket deletion post-test fails - storage.create(BucketInfo.newBuilder(bucket) - .setLifecycleRules(defaultTestDataLifecycle()) - .setSoftDeletePolicy(null) // no soft-deletion - .build()); - - //delete bucket at shutdown - Runtime.getRuntime().addShutdownHook(new Thread(() -> { - try { - RemoteStorageHelper.forceDelete(storage, bucket, 5, TimeUnit.SECONDS); - } catch (Throwable e) { - Logger.getAnonymousLogger().log(Level.WARNING, "Failed to cleanup bucket: " + bucket); - } - })); - } - } - - List defaultTestDataLifecycle() { - return Collections.singletonList(new BucketInfo.LifecycleRule( - BucketInfo.LifecycleRule.LifecycleAction.newDeleteAction(), - BucketInfo.LifecycleRule.LifecycleCondition.newBuilder().setAge(30).build())); - } - - @Deprecated //attach delete to global runtime shutdown - @Override - public void tearDown() { - } -} diff --git a/java/src/test/java/com/google/appengine/tools/mapreduce/CustomOutputTest.java b/java/src/test/java/com/google/appengine/tools/mapreduce/CustomOutputTest.java index aadf5e8a..c53fdcf1 100644 --- a/java/src/test/java/com/google/appengine/tools/mapreduce/CustomOutputTest.java +++ b/java/src/test/java/com/google/appengine/tools/mapreduce/CustomOutputTest.java @@ -12,6 +12,8 @@ import com.google.appengine.tools.pipeline.JobInfo; import com.google.appengine.tools.pipeline.JobInfo.State; +import com.google.appengine.tools.test.CloudStorageExtension; +import lombok.Getter; import org.junit.jupiter.api.Test; import java.util.ArrayList; @@ -24,6 +26,9 @@ */ public class CustomOutputTest extends EndToEndTestCase { + @Getter + String bucket; + @SuppressWarnings("serial") static class CustomWriter extends OutputWriter { final int id; @@ -98,8 +103,9 @@ public void testOutputInOrder() throws Exception { .setOutput(new CustomOutput()) .setNumReducers(17); MapReduceSettings mrSettings = new MapReduceSettings.Builder() - .setServiceAccountKey(getStorageTestHelper().getBase64EncodedServiceAccountKey()) - .setBucketName(getStorageTestHelper().getBucket()) + .setProjectId(CloudStorageExtension.getProjectId()) + .setServiceAccountKey(CloudStorageExtension.getBase64EncodedServiceAccountKey()) + .setBucketName(getBucket()) .setDatastoreHost(datastore.getOptions().getHost()) .setProjectId(datastore.getOptions().getProjectId()) .setDatabaseId(datastore.getOptions().getDatabaseId()) diff --git a/java/src/test/java/com/google/appengine/tools/mapreduce/DatastoreExtension.java b/java/src/test/java/com/google/appengine/tools/mapreduce/DatastoreExtension.java deleted file mode 100644 index 33f9b695..00000000 --- a/java/src/test/java/com/google/appengine/tools/mapreduce/DatastoreExtension.java +++ /dev/null @@ -1,97 +0,0 @@ -package com.google.appengine.tools.mapreduce; - -import com.google.cloud.datastore.Datastore; -import com.google.cloud.datastore.DatastoreOptions; -import com.google.cloud.datastore.testing.LocalDatastoreHelper; -import lombok.extern.java.Log; -import org.junit.jupiter.api.extension.*; - -import java.net.ConnectException; -import java.time.Duration; -import java.util.logging.Level; - -/** - * Junit5 extension to initialize local datastore emulator for tests - * Use it in your tests with {@code @ExtendWith(DatastoreExtension.class)} - * - * TODO: replace with setup for all the pipelines stuff?? - */ -@Log -public class DatastoreExtension implements BeforeAllCallback, AfterAllCallback, BeforeEachCallback { - - public static String TEST_DATASTORE_PROJECT_ID = "test-project"; - public static String DS_CONTEXT_KEY = "ds-emulator"; - public static String DS_OPTIONS_CONTEXT_KEY = "ds-options"; - - private LocalDatastoreHelper globalDatastoreHelper; - - @Override - public void beforeAll(ExtensionContext extensionContext) throws Exception { - globalDatastoreHelper = LocalDatastoreHelper.newBuilder() - .setStoreOnDisk(false) // can't reset if storing data disk - .setConsistency(1.0) - .build(); - globalDatastoreHelper.start(); - log.info("Datastore emulator started on port : " + globalDatastoreHelper.getPort()); - } - - @Override - public void afterAll(ExtensionContext extensionContext) throws Exception { - - int attempt = 0; - boolean stopped = false; - while (!stopped && attempt < 3) { - ++attempt; - try { - org.threeten.bp.Duration timeout = - org.threeten.bp.Duration.ofSeconds(5).multipliedBy(attempt); - globalDatastoreHelper.stop(timeout); - stopped = true; - } catch (ConnectException e) { - // don't want to kill the test, but also don't want to leave the emulator running - log.warning("DatastoreExtension : Failed to connect in order to stop datastore emulator; retrying..."); - } catch (Exception e) { - log.log(Level.WARNING, "DatastoreExtension : Failed to stop datastore emulator; retrying...", e); - } - } - log.info("Datastore emulator stopped"); - } - - @Override - public void beforeEach(ExtensionContext extensionContext) throws Exception { - globalDatastoreHelper.reset(); - log.info("Datastore emulator reset"); - DatastoreOptions options = globalDatastoreHelper.getOptions().toBuilder() - .setProjectId(TEST_DATASTORE_PROJECT_ID) - .build(); - - extensionContext.getStore(ExtensionContext.Namespace.GLOBAL).put(DS_OPTIONS_CONTEXT_KEY, options); - - Datastore datastore = options.getService(); - extensionContext.getStore(ExtensionContext.Namespace.GLOBAL).put(DS_CONTEXT_KEY, datastore); - } - - public static class ParameterResolver implements org.junit.jupiter.api.extension.ParameterResolver { - - @Override - public boolean supportsParameter(ParameterContext parameterContext, - ExtensionContext extensionContext) throws ParameterResolutionException { - return parameterContext.getParameter().getType() == Datastore.class - || parameterContext.getParameter().getType() == DatastoreOptions.class; - } - - @Override - public Object resolveParameter(ParameterContext parameterContext, - ExtensionContext extensionContext) throws ParameterResolutionException { - if (parameterContext.getParameter().getType() == Datastore.class) { - return extensionContext.getStore(ExtensionContext.Namespace.GLOBAL).get(DatastoreExtension.DS_CONTEXT_KEY); - } else if (parameterContext.getParameter().getType() == DatastoreOptions.class) { - return extensionContext.getStore(ExtensionContext.Namespace.GLOBAL).get(DatastoreExtension.DS_OPTIONS_CONTEXT_KEY); - } else { - throw new ParameterResolutionException("Unsupported parameter type: " + parameterContext.getParameter().getType()); - } - } - } - -} - diff --git a/java/src/test/java/com/google/appengine/tools/mapreduce/EndToEndTest.java b/java/src/test/java/com/google/appengine/tools/mapreduce/EndToEndTest.java index 3f6d5203..4dcc0072 100644 --- a/java/src/test/java/com/google/appengine/tools/mapreduce/EndToEndTest.java +++ b/java/src/test/java/com/google/appengine/tools/mapreduce/EndToEndTest.java @@ -37,6 +37,7 @@ import com.google.appengine.tools.mapreduce.reducers.ValueProjectionReducer; import com.google.appengine.tools.pipeline.JobRunId; import com.google.appengine.tools.pipeline.JobInfo; +import com.google.appengine.tools.test.CloudStorageExtension; import com.google.cloud.ReadChannel; import com.google.common.collect.ArrayListMultimap; import com.google.common.collect.ImmutableList; @@ -44,6 +45,7 @@ import com.google.common.collect.Lists; import com.google.common.collect.Sets; +import lombok.Getter; import lombok.RequiredArgsConstructor; import org.easymock.EasyMock; import org.junit.jupiter.api.BeforeEach; @@ -75,19 +77,21 @@ public class EndToEndTest extends EndToEndTestCase { private static final Logger log = Logger.getLogger(EndToEndTest.class.getName()); - GoogleCloudStorageFileOutput.Options cloudStorageFileOutputOptions; MapReduceSettings testSettings; + @Getter + String bucket; + @BeforeEach public void setUp() throws Exception { super.setUp(); cloudStorageFileOutputOptions = GoogleCloudStorageFileOutput.BaseOptions.defaults() - .withServiceAccountKey(getStorageTestHelper().getBase64EncodedServiceAccountKey()) - .withProjectId(getStorageTestHelper().getProjectId()); //prob not really needed .. + .withServiceAccountKey(CloudStorageExtension.getBase64EncodedServiceAccountKey()) + .withProjectId(CloudStorageExtension.getProjectId()); //prob not really needed .. testSettings = new MapReduceSettings.Builder() - .setServiceAccountKey(getStorageTestHelper().getBase64EncodedServiceAccountKey()) - .setBucketName(getStorageTestHelper().getBucket()) + .setServiceAccountKey(CloudStorageExtension.getBase64EncodedServiceAccountKey()) + .setBucketName(bucket) .build(); } @@ -191,7 +195,7 @@ public void testMapOnlyJobWithSizeSegmentedOutput() throws Exception { String fileNamePattern = "MapOnlySegmentingTestShard-%04d/file-%04d"; SizeSegmentedGoogleCloudStorageFileOutput output = - new SizeSegmentedGoogleCloudStorageFileOutput(getStorageTestHelper().getBucket(), 30, fileNamePattern, mimeType, cloudStorageFileOutputOptions); + new SizeSegmentedGoogleCloudStorageFileOutput(getBucket(), 30, fileNamePattern, mimeType, cloudStorageFileOutputOptions); MarshallingOutput op = new MarshallingOutput<>(output, Marshallers.getStringMarshaller()); @@ -691,7 +695,7 @@ public void testPassThroughToString() throws Exception { final RandomLongInput input = new RandomLongInput(10, 1); input.setSeed(0L); runTest(new MapReduceSpecification.Builder<>(input, new Mod37Mapper(), ValueProjectionReducer - .create(), new StringOutput<>(",", new GoogleCloudStorageFileOutput(getStorageTestHelper().getBucket(), "Foo-%02d", "text/plain", cloudStorageFileOutputOptions))) + .create(), new StringOutput<>(",", new GoogleCloudStorageFileOutput(bucket, "Foo-%02d", "text/plain", cloudStorageFileOutputOptions))) .setKeyMarshaller(Marshallers.getStringMarshaller()) .setValueMarshaller(Marshallers.getLongMarshaller()).setJobName("TestPassThroughToString") .build(), new Verifier() { @@ -700,7 +704,7 @@ public void verify(MapReduceResult result) throws Exc assertEquals(1, result.getOutputResult().getNumFiles()); assertEquals(10, result.getCounters().getCounter(CounterNames.MAPPER_CALLS).getValue()); GcsFilename file = result.getOutputResult().getFile(0); - ReadChannel channel = getStorageTestHelper().getStorage().reader(file.asBlobId()); + ReadChannel channel = storage.reader(file.asBlobId()); BufferedReader reader = new BufferedReader(Channels.newReader(channel, US_ASCII.newDecoder(), -1)); String line = reader.readLine(); @@ -737,7 +741,7 @@ private void applyTestPassByteBufferToGcs(boolean sliceRetry) throws Exception { builder.setKeyMarshaller(Marshallers.getByteBufferMarshaller()); builder.setValueMarshaller(Marshallers.getByteBufferMarshaller()); builder.setReducer(ValueProjectionReducer.create()); - builder.setOutput(new GoogleCloudStorageFileOutput(getStorageTestHelper().getBucket(), "fileNamePattern-%04d", + builder.setOutput(new GoogleCloudStorageFileOutput(bucket, "fileNamePattern-%04d", "application/octet-stream", (GoogleCloudStorageFileOutput.Options) cloudStorageFileOutputOptions.withSupportSliceRetries(sliceRetry))); builder.setNumReducers(2); runTest(builder.build(), new Verifier() { @@ -748,8 +752,8 @@ public void verify(MapReduceResult result) throws Exc ArrayList results = new ArrayList<>(); ByteBuffer holder = ByteBuffer.allocate(8); for (GcsFilename file : result.getOutputResult().getFiles()) { - assertEquals("application/octet-stream", getStorageTestHelper().getStorage().get(file.asBlobId()).getContentType()); - try (ReadChannel reader = getStorageTestHelper().getStorage().reader(file.asBlobId())) { + assertEquals("application/octet-stream", storage.get(file.asBlobId()).getContentType()); + try (ReadChannel reader = storage.reader(file.asBlobId())) { int read = reader.read(holder); while (read != -1) { holder.rewind(); @@ -1240,7 +1244,7 @@ public void testSideOutput() throws Exception { final int SHARD_COUNT = 3; runTest(new MapReduceSpecification.Builder<>(new ConsecutiveLongInput(0, SHARD_COUNT, SHARD_COUNT), - new SideOutputMapper(getStorageTestHelper().getBucket(), cloudStorageFileOutputOptions), KeyProjectionReducer.create(), + new SideOutputMapper(bucket, cloudStorageFileOutputOptions), KeyProjectionReducer.create(), new InMemoryOutput<>()) .setKeyMarshaller(Marshallers.getSerializationMarshaller()) .setValueMarshaller(Marshallers.getVoidMarshaller()) @@ -1260,7 +1264,7 @@ public void verify(MapReduceResult>> result) throws Excep assertEquals(SHARD_COUNT, files.size()); for (GcsFilename file : files) { ByteBuffer buf = ByteBuffer.allocate(8); - try (ReadChannel ch = getStorageTestHelper().getStorage().reader(file.asBlobId())) { + try (ReadChannel ch = storage.reader(file.asBlobId())) { assertEquals(8, ch.read(buf)); assertEquals(-1, ch.read(ByteBuffer.allocate(1))); } diff --git a/java/src/test/java/com/google/appengine/tools/mapreduce/EndToEndTestCase.java b/java/src/test/java/com/google/appengine/tools/mapreduce/EndToEndTestCase.java index 933bd6c4..710b474e 100644 --- a/java/src/test/java/com/google/appengine/tools/mapreduce/EndToEndTestCase.java +++ b/java/src/test/java/com/google/appengine/tools/mapreduce/EndToEndTestCase.java @@ -10,9 +10,6 @@ import com.google.appengine.api.taskqueue.dev.QueueStateInfo; import com.google.appengine.api.taskqueue.dev.QueueStateInfo.HeaderWrapper; import com.google.appengine.api.taskqueue.dev.QueueStateInfo.TaskStateInfo; -import com.google.appengine.tools.development.testing.LocalModulesServiceTestConfig; -import com.google.appengine.tools.development.testing.LocalServiceTestHelper; -import com.google.appengine.tools.development.testing.LocalTaskQueueTestConfig; import com.google.appengine.tools.mapreduce.impl.shardedjob.IncrementalTaskId; import com.google.appengine.tools.mapreduce.impl.shardedjob.ShardedJobHandler; import com.google.appengine.tools.mapreduce.impl.shardedjob.ShardedJobRunId; @@ -22,12 +19,14 @@ import com.google.appengine.tools.pipeline.TestUtils; import com.google.appengine.tools.pipeline.impl.servlets.PipelineServlet; import com.google.appengine.tools.pipeline.impl.servlets.TaskHandler; +import com.google.appengine.tools.test.CloudStorageExtensions; +import com.google.appengine.tools.test.PipelineSetupExtensions; import com.google.cloud.datastore.Datastore; +import com.google.cloud.storage.Storage; import com.google.common.base.CharMatcher; import lombok.Getter; import lombok.Setter; -import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import java.io.ByteArrayInputStream; @@ -44,24 +43,15 @@ import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; +@CloudStorageExtensions @PipelineSetupExtensions public abstract class EndToEndTestCase { private static final Logger logger = Logger.getLogger(EndToEndTestCase.class.getName()); - private final LocalServiceTestHelper helper = - new LocalServiceTestHelper( - new LocalTaskQueueTestConfig().setDisableAutoTaskExecution(true), - // don't think we use memcache - //new LocalMemcacheServiceTestConfig(), - new LocalModulesServiceTestConfig()); + @Setter(onMethod_ = @BeforeEach) private LocalTaskQueue taskQueue; - /** Implement in sub-classes to set system environment properties for tests. */ - protected Map getEnvAttributes() throws Exception { - return null; - } - @Getter @Setter(onMethod_ = @BeforeEach) Datastore datastore; @@ -74,33 +64,20 @@ protected Map getEnvAttributes() throws Exception { @Getter @Setter(onMethod_ = @BeforeEach) PipelineOrchestrator pipelineOrchestrator; + @Getter @Setter(onMethod_ = @BeforeEach) + Storage storage; + + // will this magically have right context? private PipelineServlet pipelineServlet = new PipelineServlet(); private MapReduceServlet mrServlet = new MapReduceServlet(); - @Getter - private CloudStorageIntegrationTestHelper storageTestHelper; - @BeforeEach public void setUp() throws Exception { - helper.setUp(); - Map envAttributes = getEnvAttributes(); - if (envAttributes != null) { - LocalServiceTestHelper.getApiProxyLocal().appendProperties(envAttributes); - } - taskQueue = LocalTaskQueueTestConfig.getLocalTaskQueue(); - // Creating files is not allowed in some test execution environments, so don't. - storageTestHelper = new CloudStorageIntegrationTestHelper(); - storageTestHelper.setUp(); - pipelineServlet.init(); mrServlet.init(); } - @AfterEach - public void tearDown() throws Exception { - helper.tearDown(); - } public ShardedJobRunId shardedJobId(String jobId) { return ShardedJobRunId.of( diff --git a/java/src/test/java/com/google/appengine/tools/mapreduce/MapReduceSpecificationTest.java b/java/src/test/java/com/google/appengine/tools/mapreduce/MapReduceSpecificationTest.java index e6ef60c9..990737d9 100644 --- a/java/src/test/java/com/google/appengine/tools/mapreduce/MapReduceSpecificationTest.java +++ b/java/src/test/java/com/google/appengine/tools/mapreduce/MapReduceSpecificationTest.java @@ -5,12 +5,12 @@ import com.google.appengine.tools.mapreduce.impl.InProcessMapReduce; import com.google.appengine.tools.pipeline.PipelineService; +import com.google.appengine.tools.test.PipelineSetupExtensions; import com.google.common.collect.ImmutableList; import lombok.Getter; import lombok.Setter; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; import java.util.ArrayList; import java.util.Arrays; diff --git a/java/src/test/java/com/google/appengine/tools/mapreduce/MapSettingsTest.java b/java/src/test/java/com/google/appengine/tools/mapreduce/MapSettingsTest.java index 20e3c5f0..a3529ecf 100644 --- a/java/src/test/java/com/google/appengine/tools/mapreduce/MapSettingsTest.java +++ b/java/src/test/java/com/google/appengine/tools/mapreduce/MapSettingsTest.java @@ -22,11 +22,11 @@ import com.google.appengine.tools.pipeline.JobSetting.OnService; import com.google.appengine.tools.pipeline.JobSetting.OnQueue; import com.google.appengine.tools.pipeline.JobSetting.StatusConsoleUrl; +import com.google.appengine.tools.test.PipelineSetupExtensions; import com.google.apphosting.api.ApiProxy; import com.google.apphosting.api.ApiProxy.Environment; import com.google.cloud.datastore.Datastore; -import com.google.cloud.datastore.Key; import org.easymock.EasyMock; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; diff --git a/java/src/test/java/com/google/appengine/tools/mapreduce/MapSpecificationTest.java b/java/src/test/java/com/google/appengine/tools/mapreduce/MapSpecificationTest.java index 61a8fb03..4592077e 100644 --- a/java/src/test/java/com/google/appengine/tools/mapreduce/MapSpecificationTest.java +++ b/java/src/test/java/com/google/appengine/tools/mapreduce/MapSpecificationTest.java @@ -2,12 +2,12 @@ import com.google.appengine.tools.mapreduce.impl.InProcessMap; import com.google.appengine.tools.pipeline.PipelineService; +import com.google.appengine.tools.test.PipelineSetupExtensions; import com.google.common.collect.ImmutableList; import lombok.Getter; import lombok.Setter; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; import java.util.ArrayList; import java.util.Arrays; diff --git a/java/src/test/java/com/google/appengine/tools/mapreduce/impl/GoogleCloudStorageMapOutputTest.java b/java/src/test/java/com/google/appengine/tools/mapreduce/impl/GoogleCloudStorageMapOutputTest.java index cf5d07c4..2276ad7a 100644 --- a/java/src/test/java/com/google/appengine/tools/mapreduce/impl/GoogleCloudStorageMapOutputTest.java +++ b/java/src/test/java/com/google/appengine/tools/mapreduce/impl/GoogleCloudStorageMapOutputTest.java @@ -7,7 +7,11 @@ import com.google.appengine.tools.mapreduce.impl.shardedjob.ShardedJobRunId; import com.google.appengine.tools.mapreduce.inputs.GoogleCloudStorageLineInput; import com.google.appengine.tools.mapreduce.outputs.GoogleCloudStorageFileOutput; +import com.google.appengine.tools.test.CloudStorageExtension; +import com.google.appengine.tools.test.CloudStorageExtensions; +import com.google.cloud.storage.Storage; import lombok.AllArgsConstructor; +import lombok.Setter; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -29,6 +33,7 @@ * Test class for {@link GoogleCloudStorageMapOutput}. * */ +@CloudStorageExtensions public class GoogleCloudStorageMapOutputTest { private static final ShardedJobRunId JOB = ShardedJobRunId.of("test-project", null, null,"JOB1"); @@ -41,20 +46,21 @@ public class GoogleCloudStorageMapOutputTest { private static final Random RND = new SecureRandom(); private final LocalServiceTestHelper helper = new LocalServiceTestHelper(); - private final CloudStorageIntegrationTestHelper cloudStorageIntegrationTestHelper = new CloudStorageIntegrationTestHelper(); @BeforeEach public void setUp() { helper.setUp(); System.setProperty(COMPONENTS_PER_COMPOSE_PROPERTY, String.valueOf(COMPONENTS_PER_COMPOSE)); - cloudStorageIntegrationTestHelper.setUp(); } + @Setter(onMethod_ = @BeforeEach) + Storage storage; + String bucket; + @AfterEach public void tearDown() { helper.tearDown(); System.clearProperty(COMPONENTS_PER_COMPOSE_PROPERTY); - cloudStorageIntegrationTestHelper.tearDown(); } @Test @@ -110,13 +116,13 @@ private static class SliceData { private void writeAndVerifyContent(SliceData... sliceData) throws IOException { GoogleCloudStorageFileOutput.BaseOptions outputOptions = GoogleCloudStorageFileOutput.BaseOptions.builder() - .serviceAccountKey(cloudStorageIntegrationTestHelper.getBase64EncodedServiceAccountKey()) - .projectId(cloudStorageIntegrationTestHelper.getProjectId()).build(); + .serviceAccountKey(CloudStorageExtension.getBase64EncodedServiceAccountKey()) + .projectId(CloudStorageExtension.getProjectId()).build(); GoogleCloudStorageLineInput.BaseOptions inputOptions = GoogleCloudStorageLineInput.BaseOptions.builder() - .serviceAccountKey(cloudStorageIntegrationTestHelper.getBase64EncodedServiceAccountKey()) + .serviceAccountKey(CloudStorageExtension.getBase64EncodedServiceAccountKey()) .build(); - GoogleCloudStorageMapOutput output = new GoogleCloudStorageMapOutput<>(cloudStorageIntegrationTestHelper.getBucket(), + GoogleCloudStorageMapOutput output = new GoogleCloudStorageMapOutput<>(this.bucket, JOB, KEY_MARSHALLER, VALUE_MARSHALLER, new Sharder() { private static final long serialVersionUID = 1L; diff --git a/java/src/test/java/com/google/appengine/tools/mapreduce/impl/handlers/MapReduceServletTest.java b/java/src/test/java/com/google/appengine/tools/mapreduce/impl/handlers/MapReduceServletTest.java index 50815fda..9846a33b 100644 --- a/java/src/test/java/com/google/appengine/tools/mapreduce/impl/handlers/MapReduceServletTest.java +++ b/java/src/test/java/com/google/appengine/tools/mapreduce/impl/handlers/MapReduceServletTest.java @@ -23,13 +23,12 @@ import static org.easymock.EasyMock.verify; import static org.junit.jupiter.api.Assertions.fail; -import com.google.appengine.tools.development.testing.LocalMemcacheServiceTestConfig; import com.google.appengine.tools.development.testing.LocalServiceTestHelper; import com.google.appengine.tools.development.testing.LocalTaskQueueTestConfig; import com.google.appengine.tools.mapreduce.MapReduceJob; import com.google.appengine.tools.mapreduce.MapReduceServlet; -import com.google.appengine.tools.mapreduce.PipelineSetupExtensions; +import com.google.appengine.tools.test.PipelineSetupExtensions; import com.google.appengine.tools.pipeline.TestUtils; import com.google.appengine.tools.pipeline.TestingTaskQueueCallback; import com.google.appengine.tools.pipeline.impl.PipelineManager; @@ -57,7 +56,7 @@ public class MapReduceServletTest{ private final LocalServiceTestHelper helper = new LocalServiceTestHelper( - new LocalTaskQueueTestConfig(), new LocalMemcacheServiceTestConfig()); + new LocalTaskQueueTestConfig()); private MapReduceServlet servlet; diff --git a/java/src/test/java/com/google/appengine/tools/mapreduce/impl/shardedjob/LockingTest.java b/java/src/test/java/com/google/appengine/tools/mapreduce/impl/shardedjob/LockingTest.java index 5a1195ea..c60d42d8 100644 --- a/java/src/test/java/com/google/appengine/tools/mapreduce/impl/shardedjob/LockingTest.java +++ b/java/src/test/java/com/google/appengine/tools/mapreduce/impl/shardedjob/LockingTest.java @@ -6,7 +6,7 @@ import com.google.appengine.api.taskqueue.dev.QueueStateInfo.TaskStateInfo; import com.google.appengine.tools.mapreduce.EndToEndTestCase; -import com.google.appengine.tools.mapreduce.PipelineSetupExtensions; +import com.google.appengine.tools.test.PipelineSetupExtensions; import com.google.appengine.tools.pipeline.impl.backend.AppEngineBackEnd; import com.google.apphosting.api.ApiProxy; import com.google.apphosting.api.ApiProxy.Environment; diff --git a/java/src/test/java/com/google/appengine/tools/mapreduce/impl/util/SerializationUtilTest.java b/java/src/test/java/com/google/appengine/tools/mapreduce/impl/util/SerializationUtilTest.java index ab3c6c39..2463398a 100644 --- a/java/src/test/java/com/google/appengine/tools/mapreduce/impl/util/SerializationUtilTest.java +++ b/java/src/test/java/com/google/appengine/tools/mapreduce/impl/util/SerializationUtilTest.java @@ -7,7 +7,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import com.google.appengine.tools.development.testing.LocalServiceTestHelper; -import com.google.appengine.tools.mapreduce.DatastoreExtension; +import com.google.appengine.tools.test.DatastoreExtension; import com.google.cloud.datastore.Datastore; import com.google.cloud.datastore.Entity; @@ -23,7 +23,6 @@ import java.io.Serializable; import java.nio.ByteBuffer; import java.util.Arrays; -import java.util.List; import java.util.Random; /** @@ -31,7 +30,6 @@ */ @ExtendWith({ DatastoreExtension.class, - //AppEngineEnvironmentExtension.class, DatastoreExtension.ParameterResolver.class, }) public class SerializationUtilTest { diff --git a/java/src/test/java/com/google/appengine/tools/mapreduce/inputs/GoogleCloudStorageLevelDbInputReaderTest.java b/java/src/test/java/com/google/appengine/tools/mapreduce/inputs/GoogleCloudStorageLevelDbInputReaderTest.java index 754b84e1..439f871d 100644 --- a/java/src/test/java/com/google/appengine/tools/mapreduce/inputs/GoogleCloudStorageLevelDbInputReaderTest.java +++ b/java/src/test/java/com/google/appengine/tools/mapreduce/inputs/GoogleCloudStorageLevelDbInputReaderTest.java @@ -1,11 +1,11 @@ package com.google.appengine.tools.mapreduce.inputs; -import com.google.appengine.tools.development.testing.LocalDatastoreServiceTestConfig; import com.google.appengine.tools.development.testing.LocalServiceTestHelper; import com.google.appengine.tools.development.testing.LocalTaskQueueTestConfig; -import com.google.appengine.tools.mapreduce.CloudStorageIntegrationTestHelper; import com.google.appengine.tools.mapreduce.GcsFilename; -import com.google.appengine.tools.mapreduce.PipelineSetupExtensions; +import com.google.appengine.tools.test.CloudStorageExtension; +import com.google.appengine.tools.test.CloudStorageExtensions; +import com.google.appengine.tools.test.PipelineSetupExtensions; import com.google.appengine.tools.mapreduce.impl.util.LevelDbConstants; import com.google.appengine.tools.mapreduce.impl.util.SerializationUtil; import com.google.appengine.tools.mapreduce.outputs.GoogleCloudStorageFileOutput; @@ -14,6 +14,8 @@ import com.google.appengine.tools.mapreduce.outputs.LevelDbOutputWriter; +import com.google.cloud.storage.Storage; +import lombok.Setter; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -36,37 +38,38 @@ /** * Tests for {@link GoogleCloudStorageLevelDbInput} */ +@CloudStorageExtensions @PipelineSetupExtensions public class GoogleCloudStorageLevelDbInputReaderTest { private static final int BLOCK_SIZE = LevelDbConstants.BLOCK_SIZE; - GcsFilename filename; - private CloudStorageIntegrationTestHelper storageHelper; private final LocalServiceTestHelper helper = new LocalServiceTestHelper( new LocalTaskQueueTestConfig()); + @Setter(onMethod_ = @BeforeEach) + Storage storage; + + String bucket; + GcsFilename filename; @BeforeEach public void setUp() throws Exception { helper.setUp(); - storageHelper = new CloudStorageIntegrationTestHelper(); - storageHelper.setUp(); - filename = new GcsFilename(storageHelper.getBucket(), "GoogleCloudStorageLevelDbInputReaderTest"); + filename = new GcsFilename(bucket, "GoogleCloudStorageLevelDbInputReaderTest"); } @AfterEach public void tearDown() throws Exception { - storageHelper.getStorage().delete(filename.asBlobId()); + storage.delete(filename.asBlobId()); helper.tearDown(); - storageHelper.tearDown(); } GoogleCloudStorageLineInput.Options inputOptions() { return GoogleCloudStorageLineInput.BaseOptions.builder() - .serviceAccountKey(storageHelper.getBase64EncodedServiceAccountKey()) + .serviceAccountKey(CloudStorageExtension.getBase64EncodedServiceAccountKey()) .bufferSize(BLOCK_SIZE * 2) .build(); } @@ -111,7 +114,7 @@ public void remove() { public void writeData(GcsFilename filename, ByteBufferGenerator gen) throws IOException { LevelDbOutputWriter writer = new GoogleCloudStorageLevelDbOutputWriter( new GoogleCloudStorageFileOutputWriter(filename, "application/leveldb", GoogleCloudStorageFileOutput.BaseOptions.defaults() - .withServiceAccountKey(storageHelper.getBase64EncodedServiceAccountKey()))); + .withServiceAccountKey(CloudStorageExtension.getBase64EncodedServiceAccountKey()))); writer.beginShard(); writer.beginSlice(); while (gen.hasNext()) { diff --git a/java/src/test/java/com/google/appengine/tools/mapreduce/inputs/GoogleCloudStorageLineInputReaderTest.java b/java/src/test/java/com/google/appengine/tools/mapreduce/inputs/GoogleCloudStorageLineInputReaderTest.java index 70b439f6..f4c43fc2 100644 --- a/java/src/test/java/com/google/appengine/tools/mapreduce/inputs/GoogleCloudStorageLineInputReaderTest.java +++ b/java/src/test/java/com/google/appengine/tools/mapreduce/inputs/GoogleCloudStorageLineInputReaderTest.java @@ -2,6 +2,8 @@ import com.google.appengine.tools.mapreduce.GcsFilename; import com.google.appengine.tools.mapreduce.impl.util.SerializationUtil; +import com.google.appengine.tools.test.CloudStorageExtension; +import com.google.appengine.tools.test.CloudStorageExtensions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -26,9 +28,9 @@ public class GoogleCloudStorageLineInputReaderTest extends GoogleCloudStorageLin @BeforeEach public void prepareFile() throws Exception { - filename = new GcsFilename(cloudStorageIntegrationTestHelper.getBucket(), FILENAME); + filename = new GcsFilename(bucket, FILENAME); fileSize = createFile(filename.getObjectName(), RECORD, RECORDS_COUNT); - inputOptions = GoogleCloudStorageLineInput.BaseOptions.defaults().withServiceAccountKey(cloudStorageIntegrationTestHelper.getBase64EncodedServiceAccountKey()); + inputOptions = GoogleCloudStorageLineInput.BaseOptions.defaults().withServiceAccountKey(CloudStorageExtension.getBase64EncodedServiceAccountKey()); } @Test diff --git a/java/src/test/java/com/google/appengine/tools/mapreduce/inputs/GoogleCloudStorageLineInputTest.java b/java/src/test/java/com/google/appengine/tools/mapreduce/inputs/GoogleCloudStorageLineInputTest.java index b981db4a..7c4b698b 100644 --- a/java/src/test/java/com/google/appengine/tools/mapreduce/inputs/GoogleCloudStorageLineInputTest.java +++ b/java/src/test/java/com/google/appengine/tools/mapreduce/inputs/GoogleCloudStorageLineInputTest.java @@ -2,6 +2,7 @@ import com.google.appengine.tools.mapreduce.GcsFilename; import com.google.appengine.tools.mapreduce.InputReader; +import com.google.appengine.tools.test.CloudStorageExtension; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -24,9 +25,9 @@ public class GoogleCloudStorageLineInputTest extends GoogleCloudStorageLineInput @BeforeEach public void setUpFile() throws Exception { - filename = new GcsFilename(cloudStorageIntegrationTestHelper.getBucket(), FILENAME); + filename = new GcsFilename(bucket, FILENAME); fileSize = createFile(filename.getObjectName(), RECORD, RECORDS_COUNT); - inputOptions = GoogleCloudStorageLineInput.BaseOptions.defaults().withServiceAccountKey(cloudStorageIntegrationTestHelper.getBase64EncodedServiceAccountKey()); + inputOptions = GoogleCloudStorageLineInput.BaseOptions.defaults().withServiceAccountKey(CloudStorageExtension.getBase64EncodedServiceAccountKey()); } @Test diff --git a/java/src/test/java/com/google/appengine/tools/mapreduce/inputs/GoogleCloudStorageLineInputTestCase.java b/java/src/test/java/com/google/appengine/tools/mapreduce/inputs/GoogleCloudStorageLineInputTestCase.java index d08f822c..0e989bcd 100644 --- a/java/src/test/java/com/google/appengine/tools/mapreduce/inputs/GoogleCloudStorageLineInputTestCase.java +++ b/java/src/test/java/com/google/appengine/tools/mapreduce/inputs/GoogleCloudStorageLineInputTestCase.java @@ -3,11 +3,13 @@ import com.google.appengine.tools.development.testing.LocalDatastoreServiceTestConfig; import com.google.appengine.tools.development.testing.LocalServiceTestHelper; -import com.google.appengine.tools.mapreduce.CloudStorageIntegrationTestHelper; +import com.google.appengine.tools.test.CloudStorageExtensions; import com.google.cloud.WriteChannel; import com.google.cloud.storage.BlobId; import com.google.cloud.storage.BlobInfo; import com.google.cloud.storage.Storage; +import lombok.Getter; +import lombok.Setter; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; @@ -15,33 +17,35 @@ import java.nio.ByteBuffer; +@CloudStorageExtensions abstract class GoogleCloudStorageLineInputTestCase { - CloudStorageIntegrationTestHelper cloudStorageIntegrationTestHelper; - private final LocalServiceTestHelper helper = new LocalServiceTestHelper( new LocalDatastoreServiceTestConfig()); @BeforeEach public void setUp() throws Exception { helper.setUp(); - cloudStorageIntegrationTestHelper = new CloudStorageIntegrationTestHelper(); - cloudStorageIntegrationTestHelper.setUp(); } + @Getter + @Setter(onMethod_ = @BeforeEach) + Storage storage; + + @Getter + String bucket; + long createFile(String filename, String record, int recordsCount) throws IOException { - Storage storage = cloudStorageIntegrationTestHelper.getStorage(); - try (WriteChannel writeChannel = storage.writer(BlobInfo.newBuilder(cloudStorageIntegrationTestHelper.getBucket(), filename).setContentType("application/bin").build())) { + try (WriteChannel writeChannel = storage.writer(BlobInfo.newBuilder(bucket, filename).setContentType("application/bin").build())) { for (int i = 0; i < recordsCount; i++) { writeChannel.write(ByteBuffer.wrap(record.getBytes())); } } - return cloudStorageIntegrationTestHelper.getStorage().get(BlobId.of(cloudStorageIntegrationTestHelper.getBucket(), filename)).getSize(); + return storage.get(BlobId.of(bucket, filename)).getSize(); } @AfterEach public void tearDown() throws Exception { helper.tearDown(); - cloudStorageIntegrationTestHelper.tearDown(); } } diff --git a/java/src/test/java/com/google/appengine/tools/mapreduce/outputs/GoogleCloudStorageFileOutputTest.java b/java/src/test/java/com/google/appengine/tools/mapreduce/outputs/GoogleCloudStorageFileOutputTest.java index 9e127940..167d2e8a 100644 --- a/java/src/test/java/com/google/appengine/tools/mapreduce/outputs/GoogleCloudStorageFileOutputTest.java +++ b/java/src/test/java/com/google/appengine/tools/mapreduce/outputs/GoogleCloudStorageFileOutputTest.java @@ -1,15 +1,15 @@ package com.google.appengine.tools.mapreduce.outputs; -import com.google.appengine.tools.mapreduce.CloudStorageIntegrationTestHelper; import com.google.appengine.tools.mapreduce.GoogleCloudStorageFileSet; import com.google.appengine.tools.mapreduce.OutputWriter; +import com.google.appengine.tools.test.CloudStorageExtension; +import com.google.appengine.tools.test.CloudStorageExtensions; import com.google.cloud.storage.Blob; import com.google.cloud.storage.BlobId; -import lombok.Getter; -import org.junit.BeforeClass; -import org.junit.jupiter.api.AfterEach; +import com.google.cloud.storage.Storage; +import lombok.Setter; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -26,6 +26,7 @@ import static com.google.cloud.MetadataConfig.getProjectId; import static org.junit.jupiter.api.Assertions.*; +@CloudStorageExtensions public class GoogleCloudStorageFileOutputTest { private static final String FILE_NAME_PATTERN = "shard-%02x"; @@ -37,33 +38,23 @@ public class GoogleCloudStorageFileOutputTest { // there will be some left over. private static final byte[] LARGE_CONTENT = new byte[(int) (1024 * 1024 * 2.5)]; - @Getter - CloudStorageIntegrationTestHelper storageIntegrationTestHelper; - - @BeforeClass - public static void setupStorage() { - - } @BeforeEach protected void setUp() throws Exception { - storageIntegrationTestHelper = new CloudStorageIntegrationTestHelper(); - storageIntegrationTestHelper.setUp(); // Filling the large_content buffer with a non-repeating but consistent pattern. Random r = new Random(0); r.nextBytes(LARGE_CONTENT); } + @Setter(onMethod_ = @BeforeEach) + Storage storage; - @AfterEach - protected void tearDown() throws Exception { - storageIntegrationTestHelper.tearDown(); - } + String bucket; @Test public void testFilesAreWritten() throws IOException { GoogleCloudStorageFileOutput creator = - new GoogleCloudStorageFileOutput(storageIntegrationTestHelper.getBucket(), FILE_NAME_PATTERN, MIME_TYPE, GoogleCloudStorageFileOutput.BaseOptions.defaults().withServiceAccountKey(storageIntegrationTestHelper.getBase64EncodedServiceAccountKey()).withProjectId(getProjectId())); + new GoogleCloudStorageFileOutput(bucket, FILE_NAME_PATTERN, MIME_TYPE, GoogleCloudStorageFileOutput.BaseOptions.defaults().withServiceAccountKey(CloudStorageExtension.getBase64EncodedServiceAccountKey()).withProjectId(getProjectId())); List> writers = creator.createWriters(NUM_SHARDS); assertEquals(NUM_SHARDS, writers.size()); beginShard(writers); @@ -77,7 +68,7 @@ public void testFilesAreWritten() throws IOException { GoogleCloudStorageFileSet files = creator.finish(writers); assertEquals(NUM_SHARDS, files.getNumFiles()); for (int i = 0; i < NUM_SHARDS; i++) { - Blob blob = storageIntegrationTestHelper.getStorage().get(BlobId.of(files.getFile(i).getBucketName(), files.getFile(i).getObjectName())); + Blob blob = storage.get(BlobId.of(files.getFile(i).getBucketName(), files.getFile(i).getObjectName())); assertNotNull(blob); assertEquals(SMALL_CONTENT.length, (long) blob.getSize()); assertEquals(MIME_TYPE, blob.getContentType()); @@ -102,7 +93,7 @@ public void testLargeSlicing() throws IOException, ClassNotFoundException { private void testSlicing(byte[] content) throws IOException, ClassNotFoundException { GoogleCloudStorageFileOutput creator = - new GoogleCloudStorageFileOutput(storageIntegrationTestHelper.getBucket(), FILE_NAME_PATTERN, MIME_TYPE, GoogleCloudStorageFileOutput.BaseOptions.defaults().withServiceAccountKey(storageIntegrationTestHelper.getBase64EncodedServiceAccountKey()).withProjectId(getProjectId())); + new GoogleCloudStorageFileOutput(bucket, FILE_NAME_PATTERN, MIME_TYPE, GoogleCloudStorageFileOutput.BaseOptions.defaults().withServiceAccountKey(CloudStorageExtension.getBase64EncodedServiceAccountKey()).withProjectId(getProjectId())); List> writers = creator.createWriters(NUM_SHARDS); assertEquals(NUM_SHARDS, writers.size()); beginShard(writers); @@ -126,11 +117,11 @@ private void testSlicing(byte[] content) throws IOException, ClassNotFoundExcept expectedContent.rewind(); ByteBuffer actualContent = ByteBuffer.allocate(content.length * 2 + 1); BlobId blobId = BlobId.of(files.getFile(i).getBucketName(), files.getFile(i).getObjectName()); - Blob blob = storageIntegrationTestHelper.getStorage().get(BlobId.of(files.getFile(i).getBucketName(), files.getFile(i).getObjectName())); + Blob blob = storage.get(BlobId.of(files.getFile(i).getBucketName(), files.getFile(i).getObjectName())); assertNotNull(blob); assertEquals(expectedContent.capacity(), (long) blob.getSize()); assertEquals(MIME_TYPE, blob.getContentType()); - try (ReadableByteChannel readChannel = storageIntegrationTestHelper.getStorage().reader(blobId)) { + try (ReadableByteChannel readChannel = storage.reader(blobId)) { int read = readChannel.read(actualContent); assertEquals(read, content.length * 2); actualContent.limit(actualContent.position()); diff --git a/java/src/test/java/com/google/appengine/tools/mapreduce/outputs/SizeSegmentedGoogleCloudStorageFileOutputTest.java b/java/src/test/java/com/google/appengine/tools/mapreduce/outputs/SizeSegmentedGoogleCloudStorageFileOutputTest.java index 485805e4..bb02c515 100644 --- a/java/src/test/java/com/google/appengine/tools/mapreduce/outputs/SizeSegmentedGoogleCloudStorageFileOutputTest.java +++ b/java/src/test/java/com/google/appengine/tools/mapreduce/outputs/SizeSegmentedGoogleCloudStorageFileOutputTest.java @@ -1,13 +1,16 @@ package com.google.appengine.tools.mapreduce.outputs; import com.google.appengine.tools.development.testing.LocalServiceTestHelper; -import com.google.appengine.tools.mapreduce.CloudStorageIntegrationTestHelper; import com.google.appengine.tools.mapreduce.GoogleCloudStorageFileSet; import com.google.appengine.tools.mapreduce.OutputWriter; import com.google.appengine.tools.mapreduce.impl.util.SerializationUtil; +import com.google.appengine.tools.test.CloudStorageExtension; +import com.google.appengine.tools.test.CloudStorageExtensions; import com.google.cloud.storage.Blob; +import com.google.cloud.storage.Storage; import lombok.Getter; +import lombok.Setter; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -21,13 +24,11 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; +@CloudStorageExtensions public class SizeSegmentedGoogleCloudStorageFileOutputTest { private final LocalServiceTestHelper helper = new LocalServiceTestHelper(); - @Getter - static CloudStorageIntegrationTestHelper cloudStorageIntegrationTestHelper; - private static final String MIME_TYPE = "application/json"; public static final String GCS_FILE_NAME_FORMAT = @@ -36,14 +37,15 @@ public class SizeSegmentedGoogleCloudStorageFileOutputTest { GoogleCloudStorageFileOutput.Options options; + @Getter + String bucket; + @Getter @Setter(onMethod_ = @BeforeEach) + Storage storage; + @BeforeEach protected void setUp() throws Exception { - helper.setUp(); - cloudStorageIntegrationTestHelper = new CloudStorageIntegrationTestHelper(); - cloudStorageIntegrationTestHelper.setUp(); - options = GoogleCloudStorageFileOutput.BaseOptions.defaults().withServiceAccountKey(cloudStorageIntegrationTestHelper.getBase64EncodedServiceAccountKey()).withProjectId(cloudStorageIntegrationTestHelper.getProjectId()); - + options = GoogleCloudStorageFileOutput.BaseOptions.defaults().withServiceAccountKey(CloudStorageExtension.getBase64EncodedServiceAccountKey()).withProjectId(CloudStorageExtension.getProjectId()); } @AfterEach @@ -56,7 +58,7 @@ public void testFilesWritten() throws IOException { int segmentSizeLimit = 10; String fileNamePattern = String.format(GCS_FILE_NAME_FORMAT, "testJob"); SizeSegmentedGoogleCloudStorageFileOutput segmenter = - new SizeSegmentedGoogleCloudStorageFileOutput(cloudStorageIntegrationTestHelper.getBucket(), segmentSizeLimit, fileNamePattern, + new SizeSegmentedGoogleCloudStorageFileOutput(bucket, segmentSizeLimit, fileNamePattern, MIME_TYPE, options); List> writers = segmenter.createWriters(5); List> finished = new ArrayList<>(); @@ -80,7 +82,7 @@ public void testFilesWritten() throws IOException { GoogleCloudStorageFileSet filesWritten = segmenter.finish(finished); assertEquals(15, filesWritten.getNumFiles()); for (int i = 0; i < filesWritten.getNumFiles(); i++) { - Blob blob = cloudStorageIntegrationTestHelper.getStorage().get(filesWritten.getFile(i).asBlobId()); + Blob blob = getStorage().get(filesWritten.getFile(i).asBlobId()); assertNotNull(blob); assertEquals(MIME_TYPE, blob.getContentType()); } @@ -90,7 +92,7 @@ public void testFilesWritten() throws IOException { public void testSegmentation() throws IOException { int segmentSizeLimit = 10; SizeSegmentedGoogleCloudStorageFileOutput segmenter = - new SizeSegmentedGoogleCloudStorageFileOutput(cloudStorageIntegrationTestHelper.getBucket(), segmentSizeLimit, "testJob/%%04d_%d", + new SizeSegmentedGoogleCloudStorageFileOutput(getBucket(), segmentSizeLimit, "testJob/%%04d_%d", MIME_TYPE, options); List> writers = segmenter.createWriters(5); int countFiles = 0; @@ -101,7 +103,7 @@ public void testSegmentation() throws IOException { GoogleCloudStorageFileSet filesWritten = segmenter.finish(writers); assertEquals(countFiles, filesWritten.getNumFiles()); for (int i = 0; i < filesWritten.getNumFiles(); i++) { - Blob blob = cloudStorageIntegrationTestHelper.getStorage().get(filesWritten.getFile(i).asBlobId()); + Blob blob = getStorage().get(filesWritten.getFile(i).asBlobId()); assertNotNull(blob); assertEquals(MIME_TYPE, blob.getContentType()); } diff --git a/java/src/test/java/com/google/appengine/tools/mapreduce/servlets/ShufflerServletTest.java b/java/src/test/java/com/google/appengine/tools/mapreduce/servlets/ShufflerServletTest.java index 466e9674..572aaff6 100644 --- a/java/src/test/java/com/google/appengine/tools/mapreduce/servlets/ShufflerServletTest.java +++ b/java/src/test/java/com/google/appengine/tools/mapreduce/servlets/ShufflerServletTest.java @@ -22,12 +22,10 @@ import com.google.appengine.api.taskqueue.QueueFactory; import com.google.appengine.tools.cloudtasktest.JakartaServletInvokingTaskCallback; import com.google.appengine.tools.development.ApiProxyLocal; -import com.google.appengine.tools.development.testing.LocalMemcacheServiceTestConfig; import com.google.appengine.tools.development.testing.LocalModulesServiceTestConfig; import com.google.appengine.tools.development.testing.LocalServiceTestHelper; import com.google.appengine.tools.development.testing.LocalTaskQueueTestConfig; import com.google.appengine.tools.mapreduce.*; -import com.google.appengine.tools.mapreduce.impl.shardedjob.ShardedJobRunId; import com.google.appengine.tools.mapreduce.impl.sort.LexicographicalComparator; import com.google.appengine.tools.mapreduce.impl.util.RequestUtils; import com.google.appengine.tools.mapreduce.inputs.GoogleCloudStorageLevelDbInputReader; @@ -40,13 +38,16 @@ import com.google.appengine.tools.pipeline.PipelineService; import com.google.appengine.tools.pipeline.di.JobRunServiceComponent; import com.google.appengine.tools.pipeline.impl.servlets.PipelineServlet; +import com.google.appengine.tools.test.CloudStorageExtension; +import com.google.appengine.tools.test.CloudStorageExtensions; +import com.google.appengine.tools.test.PipelineSetupExtensions; import com.google.apphosting.api.ApiProxy; import com.google.cloud.ReadChannel; import com.google.cloud.datastore.Datastore; +import com.google.cloud.storage.Storage; import com.google.common.collect.ImmutableMap; import com.google.common.collect.TreeMultimap; -import lombok.Getter; import lombok.Setter; import lombok.SneakyThrows; import org.junit.jupiter.api.AfterEach; @@ -70,6 +71,7 @@ /** * Tests for {@link ShufflerServlet} */ +@CloudStorageExtensions @PipelineSetupExtensions public class ShufflerServletTest { @@ -93,7 +95,6 @@ public class ShufflerServletTest { private final LocalServiceTestHelper helper = new LocalServiceTestHelper( new LocalTaskQueueTestConfig().setDisableAutoTaskExecution(false).setCallbackClass(TaskRunner.class), - new LocalMemcacheServiceTestConfig(), new LocalModulesServiceTestConfig() ); @@ -131,11 +132,11 @@ protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws Se } } - @Getter - CloudStorageIntegrationTestHelper storageIntegrationTestHelper; @Setter(onMethod_ = @BeforeEach) PipelineService pipelineService; + @Setter(onMethod_ = @BeforeEach) + Storage storage; @BeforeEach public void setUp(JobRunServiceComponent component, @@ -145,8 +146,6 @@ public void setUp(JobRunServiceComponent component, // Creating files is not allowed in some test execution environments, so don't. proxy.setProperty(LocalBlobstoreService.NO_STORAGE_PROPERTY, "true"); WAIT_ON.drainPermits(); - storageIntegrationTestHelper = new CloudStorageIntegrationTestHelper(); - storageIntegrationTestHelper.setUp(); TaskRunner.extraParamValues = Map.of(RequestUtils.Params.DATASTORE_HOST, datastore.getOptions().getHost()); @@ -188,10 +187,12 @@ private int getQueueDepth() { .size(); } + String bucket; + @SneakyThrows @Test public void testDataIsOrdered() throws InterruptedException, IOException { - ShufflerParams shufflerParams = createParams(storageIntegrationTestHelper.getBase64EncodedServiceAccountKey(), storageIntegrationTestHelper.getBucket(), 3, 2); + ShufflerParams shufflerParams = createParams(CloudStorageExtension.getBase64EncodedServiceAccountKey(), bucket, 3, 2); // for test purposes, give a manifest file name that's unique, yet known outside of the shuffle stage of the map reduce job // (in usual case, derived from the shuffle stage's job id, which isn't known outside) @@ -214,7 +215,7 @@ public void testDataIsOrdered() throws InterruptedException, IOException { @Test public void testJson() throws IOException { - ShufflerParams shufflerParams = createParams(storageIntegrationTestHelper.getBase64EncodedServiceAccountKey(), storageIntegrationTestHelper.getBucket(), 3, 2); + ShufflerParams shufflerParams = createParams(CloudStorageExtension.getBase64EncodedServiceAccountKey(), bucket, 3, 2); Marshaller marshaller = Marshallers.getGenericJsonMarshaller(ShufflerParams.class); ByteBuffer bytes = marshaller.toBytes(shufflerParams); @@ -248,7 +249,7 @@ List>> validateOrdered(ShufflerParams shuf GcsFilename manifest = ShuffleMapReduce.getManifestFile(null, shufflerParams); List outputFiles; - try (ReadChannel readChannel = storageIntegrationTestHelper.getStorage().get(manifest.asBlobId()).reader()) { + try (ReadChannel readChannel = storage.get(manifest.asBlobId()).reader()) { byte[] manifestBytes = new byte[4000]; int read = readChannel.read(ByteBuffer.wrap(manifestBytes)); String manifestContent = new String(manifestBytes, 0, read, "UTF-8"); @@ -285,7 +286,7 @@ List>> validateOrdered(ShufflerParams shuf } GoogleCloudStorageFileOutput.Options outputOptions() { return GoogleCloudStorageFileOutput.BaseOptions.defaults() - .withServiceAccountKey(storageIntegrationTestHelper.getBase64EncodedServiceAccountKey()); + .withServiceAccountKey(CloudStorageExtension.getBase64EncodedServiceAccountKey()); } private TreeMultimap writeInputFiles(ShufflerParams shufflerParams, diff --git a/java/src/test/java/com/google/appengine/tools/pipeline/BarrierTest.java b/java/src/test/java/com/google/appengine/tools/pipeline/BarrierTest.java index 9eadb013..e4807d0d 100644 --- a/java/src/test/java/com/google/appengine/tools/pipeline/BarrierTest.java +++ b/java/src/test/java/com/google/appengine/tools/pipeline/BarrierTest.java @@ -17,6 +17,7 @@ import static com.google.appengine.tools.pipeline.impl.util.GUIDGenerator.USE_SIMPLE_GUIDS_FOR_DEBUGGING; import static org.junit.jupiter.api.Assertions.assertEquals; +import com.google.appengine.tools.test.DatastoreExtension; import com.google.cloud.datastore.Key; import com.google.appengine.tools.pipeline.impl.model.Barrier; import com.google.appengine.tools.pipeline.impl.model.Slot; @@ -26,7 +27,6 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; -import org.junit.jupiter.api.AfterEach; import java.util.ArrayList; import java.util.List; diff --git a/java/src/test/java/com/google/appengine/tools/pipeline/ClientUseTest.java b/java/src/test/java/com/google/appengine/tools/pipeline/ClientUseTest.java index 7309b084..3569c14b 100644 --- a/java/src/test/java/com/google/appengine/tools/pipeline/ClientUseTest.java +++ b/java/src/test/java/com/google/appengine/tools/pipeline/ClientUseTest.java @@ -1,10 +1,9 @@ package com.google.appengine.tools.pipeline; -import com.google.appengine.tools.mapreduce.PipelineSetupExtensions; +import com.google.appengine.tools.test.PipelineSetupExtensions; import com.google.appengine.tools.pipeline.impl.PipelineManager; import com.google.appengine.tools.pipeline.impl.backend.AppEngineBackEnd; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; @PipelineSetupExtensions public class ClientUseTest { diff --git a/java/src/test/java/com/google/appengine/tools/pipeline/PipelineTest.java b/java/src/test/java/com/google/appengine/tools/pipeline/PipelineTest.java index dec5f496..792d56ef 100644 --- a/java/src/test/java/com/google/appengine/tools/pipeline/PipelineTest.java +++ b/java/src/test/java/com/google/appengine/tools/pipeline/PipelineTest.java @@ -23,7 +23,7 @@ import com.google.appengine.tools.development.testing.LocalModulesServiceTestConfig; import com.google.appengine.tools.development.testing.LocalServiceTestHelper; import com.google.appengine.tools.development.testing.LocalTaskQueueTestConfig; -import com.google.appengine.tools.mapreduce.PipelineSetupExtensions; +import com.google.appengine.tools.test.PipelineSetupExtensions; import com.google.appengine.tools.pipeline.di.JobRunServiceComponent; import com.google.appengine.tools.pipeline.impl.PipelineManager; diff --git a/java/src/test/java/com/google/appengine/tools/pipeline/RetryTest.java b/java/src/test/java/com/google/appengine/tools/pipeline/RetryTest.java index a524a667..63570b5b 100644 --- a/java/src/test/java/com/google/appengine/tools/pipeline/RetryTest.java +++ b/java/src/test/java/com/google/appengine/tools/pipeline/RetryTest.java @@ -20,7 +20,7 @@ import com.google.appengine.tools.development.testing.LocalModulesServiceTestConfig; import com.google.appengine.tools.development.testing.LocalServiceTestHelper; import com.google.appengine.tools.development.testing.LocalTaskQueueTestConfig; -import com.google.appengine.tools.mapreduce.PipelineSetupExtensions; +import com.google.appengine.tools.test.PipelineSetupExtensions; import com.google.appengine.tools.pipeline.JobSetting.BackoffFactor; import com.google.appengine.tools.pipeline.JobSetting.BackoffSeconds; import com.google.appengine.tools.pipeline.JobSetting.MaxAttempts; diff --git a/java/src/test/java/com/google/appengine/tools/pipeline/UserGuideTest.java b/java/src/test/java/com/google/appengine/tools/pipeline/UserGuideTest.java index 783eb16e..5ce325e5 100644 --- a/java/src/test/java/com/google/appengine/tools/pipeline/UserGuideTest.java +++ b/java/src/test/java/com/google/appengine/tools/pipeline/UserGuideTest.java @@ -20,7 +20,7 @@ import com.google.appengine.tools.development.testing.LocalModulesServiceTestConfig; import com.google.appengine.tools.development.testing.LocalServiceTestHelper; import com.google.appengine.tools.development.testing.LocalTaskQueueTestConfig; -import com.google.appengine.tools.mapreduce.PipelineSetupExtensions; +import com.google.appengine.tools.test.PipelineSetupExtensions; import com.google.appengine.tools.pipeline.demo.UserGuideExamples.ComplexJob; import com.google.appengine.tools.pipeline.impl.PipelineManager; diff --git a/java/src/test/java/com/google/appengine/tools/pipeline/impl/backend/AppEngineBackEndTest.java b/java/src/test/java/com/google/appengine/tools/pipeline/impl/backend/AppEngineBackEndTest.java index 1529740f..0aa921ea 100644 --- a/java/src/test/java/com/google/appengine/tools/pipeline/impl/backend/AppEngineBackEndTest.java +++ b/java/src/test/java/com/google/appengine/tools/pipeline/impl/backend/AppEngineBackEndTest.java @@ -1,6 +1,6 @@ package com.google.appengine.tools.pipeline.impl.backend; -import com.google.appengine.tools.pipeline.DatastoreExtension; +import com.google.appengine.tools.test.DatastoreExtension; import com.google.appengine.tools.pipeline.impl.model.Slot; import com.google.cloud.datastore.Blob; import com.google.cloud.datastore.Datastore; diff --git a/java/src/test/java/com/google/appengine/tools/pipeline/impl/backend/AppEngineTaskQueueTest.java b/java/src/test/java/com/google/appengine/tools/pipeline/impl/backend/AppEngineTaskQueueTest.java index b0ecda03..40dc1a2a 100644 --- a/java/src/test/java/com/google/appengine/tools/pipeline/impl/backend/AppEngineTaskQueueTest.java +++ b/java/src/test/java/com/google/appengine/tools/pipeline/impl/backend/AppEngineTaskQueueTest.java @@ -1,6 +1,6 @@ package com.google.appengine.tools.pipeline.impl.backend; -import com.google.appengine.tools.pipeline.DatastoreExtension; +import com.google.appengine.tools.test.DatastoreExtension; import com.google.cloud.datastore.Key; import com.google.appengine.api.taskqueue.TaskHandle; import com.google.appengine.tools.development.testing.LocalModulesServiceTestConfig; diff --git a/java/src/test/java/com/google/appengine/tools/test/CloudStorageExtension.java b/java/src/test/java/com/google/appengine/tools/test/CloudStorageExtension.java new file mode 100644 index 00000000..70ef3ed2 --- /dev/null +++ b/java/src/test/java/com/google/appengine/tools/test/CloudStorageExtension.java @@ -0,0 +1,167 @@ +/** + * Copyright 2025 Worklytics, Co. Licensed under the Apache License, Version 2.0: http://www.apache.org/licenses/LICENSE-2.0 + */ + +package com.google.appengine.tools.test; + + +import com.google.auth.Credentials; +import com.google.auth.oauth2.ServiceAccountCredentials; +import com.google.cloud.storage.BucketInfo; +import com.google.cloud.storage.Storage; +import com.google.cloud.storage.StorageOptions; +import com.google.cloud.storage.testing.RemoteStorageHelper; +import lombok.SneakyThrows; +import org.junit.jupiter.api.extension.*; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.lang.reflect.Field; +import java.lang.reflect.Parameter; +import java.util.*; +import java.util.concurrent.TimeUnit; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * as of Apr 2020, no gcs emulator https://cloud.google.com/sdk/gcloud/reference/beta/emulators + * so this extension creates a temp bucket in target project + * + * @see "https://googleapis.dev/java/google-cloud-storage/1.106.0/com/google/cloud/storage/testing/RemoteStorageHelper.html" + * + * Auth in one of two ways: + * 1) machine has default credentials; then will use eitther default project, or one set in env var named CI_PROJECT + * 2) provide a service account credentisl + * eg, + * 1. create SA in target project; give it "Storage Admin" role + * 2. cat ~/Downloads/worklytics-ci-111242f427df.json | base64 + * 3. set output of that as your env variable + * - in IntelliJ, set this via RunConfigurations --> Env Variables. + * - in GitHub, set it as via repo --> Settings --> Secrets so it can be utilized in workflows + */ +public class CloudStorageExtension implements BeforeAllCallback, BeforeEachCallback { + + public static final String CI_PROJECT_ENV_VAR = "CI_PROJECT"; + public static final String DEFAULT_PROJECT_ID = "worklytics-ci"; + + public static final String KEY_ENV_VAR = "CI_SERVICE_ACCOUNT_KEY"; + + public static String getBase64EncodedServiceAccountKey() { + return System.getenv(KEY_ENV_VAR); + } + + static Optional getServiceAccountCredentials() { + return Optional.ofNullable(getBase64EncodedServiceAccountKey()) + .map(keyVar -> { + String base64EncodedServiceAccountKey = keyVar.trim(); + String jsonKey = new String(Base64.getDecoder().decode(base64EncodedServiceAccountKey.getBytes())); + try { + return ((ServiceAccountCredentials) ServiceAccountCredentials.fromStream(new ByteArrayInputStream(jsonKey.getBytes()))); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + } + + public static String getProjectId() { + return getServiceAccountCredentials().map(ServiceAccountCredentials::getProjectId) + .orElseGet(CloudStorageExtension::fallbackProiectId); + } + + static private String fallbackProiectId() { + return Optional.ofNullable(System.getenv(CI_PROJECT_ENV_VAR)).orElse(DEFAULT_PROJECT_ID); + } + + @Override + public void beforeAll(ExtensionContext context) throws Exception { + String keyVar = getBase64EncodedServiceAccountKey(); + Credentials credentials; + String projectId; + if (keyVar == null) { + //attempt w default credentials + credentials = StorageOptions.getDefaultInstance().getCredentials(); + + //TODO: more elegant solution? weirdness seems to happen if mix projects; credentials' project + // isn't exposed to java code via any public interface; yet bucket is created in the project + // to which the credentials default project is set + projectId = fallbackProiectId(); + //throw new IllegalStateException("Must set environment variable " + KEY_ENV_VAR + " as base64 encoded service account key to use for storage integration tests"); + } else { + credentials = getServiceAccountCredentials().get(); + projectId = ((ServiceAccountCredentials) credentials).getProjectId(); + } + + Storage storage = StorageOptions.newBuilder() + .setCredentials(credentials) + .setProjectId(projectId) + .build().getService(); + + context.getStore(ExtensionContext.Namespace.create(CloudStorageExtension.class)).put(Storage.class, storage); + + String bucket = context.getStore(ExtensionContext.Namespace.create(CloudStorageExtension.class)).get("bucket", String.class); + if (bucket == null) { + bucket = RemoteStorageHelper.generateBucketName(); + context.getStore(ExtensionContext.Namespace.create(CloudStorageExtension.class)).put("bucket", bucket); + + //avoid test data being retained forever, even if bucket deletion post-test fails + storage.create(BucketInfo.newBuilder(bucket) + .setLifecycleRules(defaultTestDataLifecycle()) + .setSoftDeletePolicy(null) // no soft-deletion + .build()); + + //delete bucket at shutdown + final String bucketToDelete = bucket; + Runtime.getRuntime().addShutdownHook(new Thread(() -> { + try { + RemoteStorageHelper.forceDelete(storage, bucketToDelete, 5, TimeUnit.SECONDS); + } catch (Throwable e) { + Logger.getAnonymousLogger().log(Level.WARNING, "Failed to cleanup bucket: " + bucketToDelete); + } + })); + + } + } + + /** + * file any String-type parameter named 'bucket' with the random bucket name + * + * @param context the current extension context; never {@code null} + */ + @SneakyThrows + @Override + public void beforeEach(ExtensionContext context) { + String bucket = context.getStore(ExtensionContext.Namespace.create(CloudStorageExtension.class)).get("bucket", String.class); + Field f = Arrays.stream(context.getRequiredTestInstance().getClass().getDeclaredFields()) + .filter(field -> field.getType().equals(String.class) && field.getName().equals("bucket")) + .findAny() + .orElse(null); + if (f != null) { + f.setAccessible(true); + f.set(context.getRequiredTestInstance(), bucket); + } + } + + static List defaultTestDataLifecycle() { + return Collections.singletonList(new BucketInfo.LifecycleRule( + BucketInfo.LifecycleRule.LifecycleAction.newDeleteAction(), + BucketInfo.LifecycleRule.LifecycleCondition.newBuilder().setAge(30).build())); + } + + /** + * files any Storage-type parameter in the test class with the Storage instance created in the beforeAll method + */ + public static class ParameterResolver implements org.junit.jupiter.api.extension.ParameterResolver { + + @Override + public Object resolveParameter(ParameterContext parameterContext, ExtensionContext extensionContext) throws ParameterResolutionException { + return extensionContext.getStore(ExtensionContext.Namespace.create(CloudStorageExtension.class)).get(Storage.class); + } + + @Override + public boolean supportsParameter(ParameterContext parameterContext, ExtensionContext extensionContext) throws ParameterResolutionException { + Parameter parameter = parameterContext.getParameter(); + return Storage.class.equals(parameter.getType()); + } + } + +} \ No newline at end of file diff --git a/java/src/test/java/com/google/appengine/tools/test/CloudStorageExtensions.java b/java/src/test/java/com/google/appengine/tools/test/CloudStorageExtensions.java new file mode 100644 index 00000000..0af61539 --- /dev/null +++ b/java/src/test/java/com/google/appengine/tools/test/CloudStorageExtensions.java @@ -0,0 +1,23 @@ +/** + * Copyright 2025 Worklytics, Co. + * Licensed under the Apache License, Version 2.0: http://www.apache.org/licenses/LICENSE-2.0 + */ +package com.google.appengine.tools.test; + +import org.junit.jupiter.api.extension.*; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +@Target({ ElementType.TYPE, ElementType.METHOD }) +@Retention(RetentionPolicy.RUNTIME) +@ExtendWith({ + CloudStorageExtension.class, + CloudStorageExtension.ParameterResolver.class, +}) +public @interface CloudStorageExtensions { + +} + diff --git a/java/src/test/java/com/google/appengine/tools/test/CloudTasksExtension.java b/java/src/test/java/com/google/appengine/tools/test/CloudTasksExtension.java new file mode 100644 index 00000000..713d3cb9 --- /dev/null +++ b/java/src/test/java/com/google/appengine/tools/test/CloudTasksExtension.java @@ -0,0 +1,60 @@ +package com.google.appengine.tools.test; + +import com.google.appengine.api.taskqueue.dev.LocalTaskQueue; +import com.google.appengine.tools.development.testing.LocalModulesServiceTestConfig; +import com.google.appengine.tools.development.testing.LocalServiceTestHelper; +import com.google.appengine.tools.development.testing.LocalTaskQueueTestConfig; +import org.junit.jupiter.api.extension.*; + +import java.lang.reflect.Parameter; + +/** + * Copyright 2025 Worklytics, Co. + * + * Licensed under the Apache License, Version 2.0: http://www.apache.org/licenses/LICENSE-2.0 + */ +public class CloudTasksExtension implements BeforeAllCallback, BeforeEachCallback, AfterEachCallback { + + @Override + public void beforeAll(ExtensionContext context) throws Exception { + LocalServiceTestHelper helper = + new LocalServiceTestHelper( + new LocalTaskQueueTestConfig().setDisableAutoTaskExecution(true), + new LocalModulesServiceTestConfig() //yeah, this is still here ... + ); + context.getStore(ExtensionContext.Namespace.create(getClass())).put(LocalServiceTestHelper.class, helper); + } + + @Override + public void beforeEach(ExtensionContext context) throws Exception { + LocalServiceTestHelper helper = + (LocalServiceTestHelper) context.getStore(ExtensionContext.Namespace.create(getClass())).get(LocalServiceTestHelper.class); + + helper.setUp(); + + LocalTaskQueue taskQueue = LocalTaskQueueTestConfig.getLocalTaskQueue(); + + context.getStore(ExtensionContext.Namespace.create(getClass())).put(LocalTaskQueue.class, taskQueue); + } + + @Override + public void afterEach(ExtensionContext context) throws Exception { + LocalServiceTestHelper helper = + (LocalServiceTestHelper) context.getStore(ExtensionContext.Namespace.create(getClass())).get(LocalServiceTestHelper.class); + + helper.tearDown(); + } + + public static class ParameterResolver implements org.junit.jupiter.api.extension.ParameterResolver { + @Override + public Object resolveParameter(ParameterContext parameterContext, ExtensionContext extensionContext) throws ParameterResolutionException { + return extensionContext.getStore(ExtensionContext.Namespace.create(CloudTasksExtension.class)).get(LocalTaskQueue.class ); + } + + @Override + public boolean supportsParameter(ParameterContext parameterContext, ExtensionContext extensionContext) throws ParameterResolutionException { + Parameter parameter = parameterContext.getParameter(); + return LocalTaskQueue.class.equals(parameter.getType()); + } + } +} diff --git a/java/src/test/java/com/google/appengine/tools/pipeline/DatastoreExtension.java b/java/src/test/java/com/google/appengine/tools/test/DatastoreExtension.java similarity index 94% rename from java/src/test/java/com/google/appengine/tools/pipeline/DatastoreExtension.java rename to java/src/test/java/com/google/appengine/tools/test/DatastoreExtension.java index cafae64b..755a826f 100644 --- a/java/src/test/java/com/google/appengine/tools/pipeline/DatastoreExtension.java +++ b/java/src/test/java/com/google/appengine/tools/test/DatastoreExtension.java @@ -1,4 +1,8 @@ -package com.google.appengine.tools.pipeline; +/** + * Copyright 2025 Worklytics, Co. + * Licensed under the Apache License, Version 2.0: http://www.apache.org/licenses/LICENSE-2.0 + */ +package com.google.appengine.tools.test; import com.google.cloud.datastore.Datastore; import com.google.cloud.datastore.DatastoreOptions; @@ -7,7 +11,6 @@ import org.junit.jupiter.api.extension.*; import java.net.ConnectException; -import java.time.Duration; import java.util.logging.Level; /** diff --git a/java/src/test/java/com/google/appengine/tools/mapreduce/PipelineSetupExtensions.java b/java/src/test/java/com/google/appengine/tools/test/PipelineSetupExtensions.java similarity index 97% rename from java/src/test/java/com/google/appengine/tools/mapreduce/PipelineSetupExtensions.java rename to java/src/test/java/com/google/appengine/tools/test/PipelineSetupExtensions.java index 5d518f41..f323bb8c 100644 --- a/java/src/test/java/com/google/appengine/tools/mapreduce/PipelineSetupExtensions.java +++ b/java/src/test/java/com/google/appengine/tools/test/PipelineSetupExtensions.java @@ -1,4 +1,8 @@ -package com.google.appengine.tools.mapreduce; +/** + * Copyright 2025 Worklytics, Co. + * Licensed under the Apache License, Version 2.0: http://www.apache.org/licenses/LICENSE-2.0 + */ +package com.google.appengine.tools.test; import com.google.appengine.tools.mapreduce.impl.shardedjob.ShardedJobRunner; import com.google.appengine.tools.pipeline.*;