From 9b27022610c21f2812ab22eb0974e3d47e69e194 Mon Sep 17 00:00:00 2001 From: Ross McKay Date: Thu, 31 Dec 2020 19:18:28 +1100 Subject: [PATCH 1/3] add missing Multipart classes from AWS PHP SDK --- .../src/Multipart/AbstractUploadManager.php | 321 ++++++++++++++++++ .../src/Multipart/AbstractUploader.php | 129 +++++++ .../aws-sdk-php/src/Multipart/UploadState.php | 145 ++++++++ 3 files changed, 595 insertions(+) create mode 100644 vendor/aws/aws-sdk-php/src/Multipart/AbstractUploadManager.php create mode 100644 vendor/aws/aws-sdk-php/src/Multipart/AbstractUploader.php create mode 100644 vendor/aws/aws-sdk-php/src/Multipart/UploadState.php diff --git a/vendor/aws/aws-sdk-php/src/Multipart/AbstractUploadManager.php b/vendor/aws/aws-sdk-php/src/Multipart/AbstractUploadManager.php new file mode 100644 index 00000000..2c7897c8 --- /dev/null +++ b/vendor/aws/aws-sdk-php/src/Multipart/AbstractUploadManager.php @@ -0,0 +1,321 @@ + null, + 'state' => null, + 'concurrency' => self::DEFAULT_CONCURRENCY, + 'prepare_data_source' => null, + 'before_initiate' => null, + 'before_upload' => null, + 'before_complete' => null, + 'exception_class' => 'Aws\Exception\MultipartUploadException', + ]; + + /** @var Client Client used for the upload. */ + protected $client; + + /** @var array Configuration used to perform the upload. */ + protected $config; + + /** @var array Service-specific information about the upload workflow. */ + protected $info; + + /** @var PromiseInterface Promise that represents the multipart upload. */ + protected $promise; + + /** @var UploadState State used to manage the upload. */ + protected $state; + + /** + * @param Client $client + * @param array $config + */ + public function __construct(Client $client, array $config = []) + { + $this->client = $client; + $this->info = $this->loadUploadWorkflowInfo(); + $this->config = $config + self::$defaultConfig; + $this->state = $this->determineState(); + } + + /** + * Returns the current state of the upload + * + * @return UploadState + */ + public function getState() + { + return $this->state; + } + + /** + * Upload the source using multipart upload operations. + * + * @return Result The result of the CompleteMultipartUpload operation. + * @throws \LogicException if the upload is already complete or aborted. + * @throws MultipartUploadException if an upload operation fails. + */ + public function upload() + { + return $this->promise()->wait(); + } + + /** + * Upload the source asynchronously using multipart upload operations. + * + * @return PromiseInterface + */ + public function promise() + { + if ($this->promise) { + return $this->promise; + } + + return $this->promise = Promise\coroutine(function () { + // Initiate the upload. + if ($this->state->isCompleted()) { + throw new \LogicException('This multipart upload has already ' + . 'been completed or aborted.' + ); + } + + if (!$this->state->isInitiated()) { + // Execute the prepare callback. + if (is_callable($this->config["prepare_data_source"])) { + $this->config["prepare_data_source"](); + } + + $result = (yield $this->execCommand('initiate', $this->getInitiateParams())); + $this->state->setUploadId( + $this->info['id']['upload_id'], + $result[$this->info['id']['upload_id']] + ); + $this->state->setStatus(UploadState::INITIATED); + } + + // Create a command pool from a generator that yields UploadPart + // commands for each upload part. + $resultHandler = $this->getResultHandler($errors); + $commands = new CommandPool( + $this->client, + $this->getUploadCommands($resultHandler), + [ + 'concurrency' => $this->config['concurrency'], + 'before' => $this->config['before_upload'], + ] + ); + + // Execute the pool of commands concurrently, and process errors. + yield $commands->promise(); + if ($errors) { + throw new $this->config['exception_class']($this->state, $errors); + } + + // Complete the multipart upload. + yield $this->execCommand('complete', $this->getCompleteParams()); + $this->state->setStatus(UploadState::COMPLETED); + })->otherwise($this->buildFailureCatch()); + } + + private function transformException($e) + { + // Throw errors from the operations as a specific Multipart error. + if ($e instanceof AwsException) { + $e = new $this->config['exception_class']($this->state, $e); + } + throw $e; + } + + private function buildFailureCatch() + { + if (interface_exists("Throwable")) { + return function (\Throwable $e) { + return $this->transformException($e); + }; + } else { + return function (\Exception $e) { + return $this->transformException($e); + }; + } + } + + protected function getConfig() + { + return $this->config; + } + + /** + * Provides service-specific information about the multipart upload + * workflow. + * + * This array of data should include the keys: 'command', 'id', and 'part_num'. + * + * @return array + */ + abstract protected function loadUploadWorkflowInfo(); + + /** + * Determines the part size to use for upload parts. + * + * Examines the provided partSize value and the source to determine the + * best possible part size. + * + * @throws \InvalidArgumentException if the part size is invalid. + * + * @return int + */ + abstract protected function determinePartSize(); + + /** + * Uses information from the Command and Result to determine which part was + * uploaded and mark it as uploaded in the upload's state. + * + * @param CommandInterface $command + * @param ResultInterface $result + */ + abstract protected function handleResult( + CommandInterface $command, + ResultInterface $result + ); + + /** + * Gets the service-specific parameters used to initiate the upload. + * + * @return array + */ + abstract protected function getInitiateParams(); + + /** + * Gets the service-specific parameters used to complete the upload. + * + * @return array + */ + abstract protected function getCompleteParams(); + + /** + * Based on the config and service-specific workflow info, creates a + * `Promise` for an `UploadState` object. + * + * @return PromiseInterface A `Promise` that resolves to an `UploadState`. + */ + private function determineState() + { + // If the state was provided via config, then just use it. + if ($this->config['state'] instanceof UploadState) { + return $this->config['state']; + } + + // Otherwise, construct a new state from the provided identifiers. + $required = $this->info['id']; + $id = [$required['upload_id'] => null]; + unset($required['upload_id']); + foreach ($required as $key => $param) { + if (!$this->config[$key]) { + throw new IAE('You must provide a value for "' . $key . '" in ' + . 'your config for the MultipartUploader for ' + . $this->client->getApi()->getServiceFullName() . '.'); + } + $id[$param] = $this->config[$key]; + } + $state = new UploadState($id); + $state->setPartSize($this->determinePartSize()); + + return $state; + } + + /** + * Executes a MUP command with all of the parameters for the operation. + * + * @param string $operation Name of the operation. + * @param array $params Service-specific params for the operation. + * + * @return PromiseInterface + */ + protected function execCommand($operation, array $params) + { + // Create the command. + $command = $this->client->getCommand( + $this->info['command'][$operation], + $params + $this->state->getId() + ); + + // Execute the before callback. + if (is_callable($this->config["before_{$operation}"])) { + $this->config["before_{$operation}"]($command); + } + + // Execute the command asynchronously and return the promise. + return $this->client->executeAsync($command); + } + + /** + * Returns a middleware for processing responses of part upload operations. + * + * - Adds an onFulfilled callback that calls the service-specific + * handleResult method on the Result of the operation. + * - Adds an onRejected callback that adds the error to an array of errors. + * - Has a passedByRef $errors arg that the exceptions get added to. The + * caller should use that &$errors array to do error handling. + * + * @param array $errors Errors from upload operations are added to this. + * + * @return callable + */ + protected function getResultHandler(&$errors = []) + { + return function (callable $handler) use (&$errors) { + return function ( + CommandInterface $command, + RequestInterface $request = null + ) use ($handler, &$errors) { + return $handler($command, $request)->then( + function (ResultInterface $result) use ($command) { + $this->handleResult($command, $result); + return $result; + }, + function (AwsException $e) use (&$errors) { + $errors[$e->getCommand()[$this->info['part_num']]] = $e; + return new Result(); + } + ); + }; + }; + } + + /** + * Creates a generator that yields part data for the upload's source. + * + * Yields associative arrays of parameters that are ultimately merged in + * with others to form the complete parameters of a command. This can + * include the Body parameter, which is a limited stream (i.e., a Stream + * object, decorated with a LimitStream). + * + * @param callable $resultHandler + * + * @return \Generator + */ + abstract protected function getUploadCommands(callable $resultHandler); +} diff --git a/vendor/aws/aws-sdk-php/src/Multipart/AbstractUploader.php b/vendor/aws/aws-sdk-php/src/Multipart/AbstractUploader.php new file mode 100644 index 00000000..766fb160 --- /dev/null +++ b/vendor/aws/aws-sdk-php/src/Multipart/AbstractUploader.php @@ -0,0 +1,129 @@ +source = $this->determineSource($source); + parent::__construct($client, $config); + } + + /** + * Create a stream for a part that starts at the current position and + * has a length of the upload part size (or less with the final part). + * + * @param Stream $stream + * + * @return Psr7\LimitStream + */ + protected function limitPartStream(Stream $stream) + { + // Limit what is read from the stream to the part size. + return new Psr7\LimitStream( + $stream, + $this->state->getPartSize(), + $this->source->tell() + ); + } + + protected function getUploadCommands(callable $resultHandler) + { + // Determine if the source can be seeked. + $seekable = $this->source->isSeekable() + && $this->source->getMetadata('wrapper_type') === 'plainfile'; + + for ($partNumber = 1; $this->isEof($seekable); $partNumber++) { + // If we haven't already uploaded this part, yield a new part. + if (!$this->state->hasPartBeenUploaded($partNumber)) { + $partStartPos = $this->source->tell(); + if (!($data = $this->createPart($seekable, $partNumber))) { + break; + } + $command = $this->client->getCommand( + $this->info['command']['upload'], + $data + $this->state->getId() + ); + $command->getHandlerList()->appendSign($resultHandler, 'mup'); + yield $command; + if ($this->source->tell() > $partStartPos) { + continue; + } + } + + // Advance the source's offset if not already advanced. + if ($seekable) { + $this->source->seek(min( + $this->source->tell() + $this->state->getPartSize(), + $this->source->getSize() + )); + } else { + $this->source->read($this->state->getPartSize()); + } + } + } + + /** + * Generates the parameters for an upload part by analyzing a range of the + * source starting from the current offset up to the part size. + * + * @param bool $seekable + * @param int $number + * + * @return array|null + */ + abstract protected function createPart($seekable, $number); + + /** + * Checks if the source is at EOF. + * + * @param bool $seekable + * + * @return bool + */ + private function isEof($seekable) + { + return $seekable + ? $this->source->tell() < $this->source->getSize() + : !$this->source->eof(); + } + + /** + * Turns the provided source into a stream and stores it. + * + * If a string is provided, it is assumed to be a filename, otherwise, it + * passes the value directly to `Psr7\stream_for()`. + * + * @param mixed $source + * + * @return Stream + */ + private function determineSource($source) + { + // Use the contents of a file as the data source. + if (is_string($source)) { + $source = Psr7\try_fopen($source, 'r'); + } + + // Create a source stream. + $stream = Psr7\stream_for($source); + if (!$stream->isReadable()) { + throw new IAE('Source stream must be readable.'); + } + + return $stream; + } +} diff --git a/vendor/aws/aws-sdk-php/src/Multipart/UploadState.php b/vendor/aws/aws-sdk-php/src/Multipart/UploadState.php new file mode 100644 index 00000000..4108c4f1 --- /dev/null +++ b/vendor/aws/aws-sdk-php/src/Multipart/UploadState.php @@ -0,0 +1,145 @@ +id = $id; + } + + /** + * Get the upload's ID, which is a tuple of parameters that can uniquely + * identify the upload. + * + * @return array + */ + public function getId() + { + return $this->id; + } + + /** + * Set's the "upload_id", or 3rd part of the upload's ID. This typically + * only needs to be done after initiating an upload. + * + * @param string $key The param key of the upload_id. + * @param string $value The param value of the upload_id. + */ + public function setUploadId($key, $value) + { + $this->id[$key] = $value; + } + + /** + * Get the part size. + * + * @return int + */ + public function getPartSize() + { + return $this->partSize; + } + + /** + * Set the part size. + * + * @param $partSize int Size of upload parts. + */ + public function setPartSize($partSize) + { + $this->partSize = $partSize; + } + + /** + * Marks a part as being uploaded. + * + * @param int $partNumber The part number. + * @param array $partData Data from the upload operation that needs to be + * recalled during the complete operation. + */ + public function markPartAsUploaded($partNumber, array $partData = []) + { + $this->uploadedParts[$partNumber] = $partData; + } + + /** + * Returns whether a part has been uploaded. + * + * @param int $partNumber The part number. + * + * @return bool + */ + public function hasPartBeenUploaded($partNumber) + { + return isset($this->uploadedParts[$partNumber]); + } + + /** + * Returns a sorted list of all the uploaded parts. + * + * @return array + */ + public function getUploadedParts() + { + ksort($this->uploadedParts); + + return $this->uploadedParts; + } + + /** + * Set the status of the upload. + * + * @param int $status Status is an integer code defined by the constants + * CREATED, INITIATED, and COMPLETED on this class. + */ + public function setStatus($status) + { + $this->status = $status; + } + + /** + * Determines whether the upload state is in the INITIATED status. + * + * @return bool + */ + public function isInitiated() + { + return $this->status === self::INITIATED; + } + + /** + * Determines whether the upload state is in the COMPLETED status. + * + * @return bool + */ + public function isCompleted() + { + return $this->status === self::COMPLETED; + } +} From fb808651a5311159fdce4a659ab8a55534fcc985 Mon Sep 17 00:00:00 2001 From: Ross McKay Date: Thu, 31 Dec 2020 19:18:51 +1100 Subject: [PATCH 2/3] use MultipartUploader to upload files to Amazon S3 --- inc/class-destination-s3.php | 93 ++++-------------------------------- 1 file changed, 8 insertions(+), 85 deletions(-) diff --git a/inc/class-destination-s3.php b/inc/class-destination-s3.php index 3359c7fc..eeae2fa4 100755 --- a/inc/class-destination-s3.php +++ b/inc/class-destination-s3.php @@ -963,98 +963,21 @@ public function job_run_archive( BackWPup_Job $job_object ) { } } else { // Prepare Upload - if ( $file_handle = fopen( $job_object->backup_folder . $job_object->backup_file, 'rb' ) ) { - fseek( $file_handle, $job_object->substeps_done ); + if ( is_readable( $job_object->backup_folder . $job_object->backup_file ) ) { + $uploader = new Aws\S3\MultipartUploader($s3, $job_object->backup_folder . $job_object->backup_file, [ + 'bucket' => $job_object->job['s3bucket'], + 'key' => $job_object->job['s3dir'] . $job_object->backup_file, + ]); try { - - if ( empty ( $job_object->steps_data[ $job_object->step_working ]['UploadId'] ) ) { - $args = array( - 'ACL' => 'private', - 'Bucket' => $job_object->job['s3bucket'], - 'ContentType' => MimeTypeExtractor::fromFilePath( $job_object->backup_folder . $job_object->backup_file ), - 'Key' => $job_object->job['s3dir'] . $job_object->backup_file, - ); - if ( ! empty( $job_object->job['s3ssencrypt'] ) ) { - $args['ServerSideEncryption'] = $job_object->job['s3ssencrypt']; - } - if ( ! empty( $job_object->job['s3storageclass'] ) ) { - $args['StorageClass'] = empty( $job_object->job['s3storageclass'] ) ? '' : $job_object->job['s3storageclass']; - } - - $upload = $s3->createMultipartUpload( $args ); - - $job_object->steps_data[ $job_object->step_working ]['UploadId'] = $upload->get( 'UploadId' ); - $job_object->steps_data[ $job_object->step_working ]['Parts'] = array(); - $job_object->steps_data[ $job_object->step_working ]['Part'] = 1; - } - - while ( ! feof( $file_handle ) ) { - $chunk_upload_start = microtime( true ); - $part_data = fread( $file_handle, 1048576 * 5 ); //5MB Minimum part size - $part = $s3->uploadPart( array( - 'Bucket' => $job_object->job['s3bucket'], - 'UploadId' => $job_object->steps_data[ $job_object->step_working ]['UploadId'], - 'Key' => $job_object->job['s3dir'] . $job_object->backup_file, - 'PartNumber' => $job_object->steps_data[ $job_object->step_working ]['Part'], - 'Body' => $part_data, - ) ); - $chunk_upload_time = microtime( true ) - $chunk_upload_start; - $job_object->substeps_done = $job_object->substeps_done + strlen( $part_data ); - $job_object->steps_data[ $job_object->step_working ]['Parts'][] = array( - 'ETag' => $part->get( 'ETag' ), - 'PartNumber' => $job_object->steps_data[ $job_object->step_working ]['Part'], - ); - $job_object->steps_data[ $job_object->step_working ]['Part'] ++; - $time_remaining = $job_object->do_restart_time(); - if ( $time_remaining < $chunk_upload_time ) { - $job_object->do_restart_time( true ); - } - $job_object->update_working_data(); - } - - $parts = $s3->listParts(array( - 'Bucket' => $job_object->job['s3bucket'], - 'Key' => $job_object->job['s3dir'] . $job_object->backup_file, - 'UploadId' => $job_object->steps_data[ $job_object->step_working ]['UploadId'] - )); - - $s3->completeMultipartUpload( array( - 'Bucket' => $job_object->job['s3bucket'], - 'UploadId' => $job_object->steps_data[ $job_object->step_working ]['UploadId'], - 'MultipartUpload' => array( - 'Parts' => $parts['Parts'], - ), - 'Key' => $job_object->job['s3dir'] . $job_object->backup_file, - ) ); - - } catch ( Exception $e ) { - $errorMessage = $e->getMessage(); - if ( $e instanceof Aws\Exception\AwsException ) { - $errorMessage = $e->getAwsErrorMessage(); - } + $result = $uploader->upload(); + } catch ( Aws\Exception\MultipartUploadException $e ) { + $errorMessage = $e->getAwsErrorMessage(); $job_object->log( E_USER_ERROR, sprintf( __( 'S3 Service API: %s', 'backwpup' ), $errorMessage ), $e->getFile(), $e->getLine() ); - if ( ! empty( $job_object->steps_data[ $job_object->step_working ]['uploadId'] ) ) { - $s3->abortMultipartUpload( array( - 'Bucket' => $job_object->job['s3bucket'], - 'UploadId' => $job_object->steps_data[ $job_object->step_working ]['uploadId'], - 'Key' => $job_object->job['s3dir'] . $job_object->backup_file, - ) ); - } - unset( $job_object->steps_data[ $job_object->step_working ]['UploadId'] ); - unset( $job_object->steps_data[ $job_object->step_working ]['Parts'] ); - unset( $job_object->steps_data[ $job_object->step_working ]['Part'] ); - $job_object->substeps_done = 0; - if ( is_resource( $file_handle ) ) { - fclose( $file_handle ); - } - - return false; } - fclose( $file_handle ); } else { $job_object->log( __( 'Can not open source file for transfer.', 'backwpup' ), E_USER_ERROR ); From c9076f70c82674ab4a2215f2ac4ce99d76149d30 Mon Sep 17 00:00:00 2001 From: Ross McKay Date: Thu, 7 Jan 2021 16:09:02 +1100 Subject: [PATCH 3/3] handle multipart exceptions correctly could be something other than an AwsException --- inc/class-destination-s3.php | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/inc/class-destination-s3.php b/inc/class-destination-s3.php index eeae2fa4..0ce48253 100755 --- a/inc/class-destination-s3.php +++ b/inc/class-destination-s3.php @@ -971,8 +971,11 @@ public function job_run_archive( BackWPup_Job $job_object ) { try { $result = $uploader->upload(); - } catch ( Aws\Exception\MultipartUploadException $e ) { - $errorMessage = $e->getAwsErrorMessage(); + } catch ( Exception $e ) { + $errorMessage = $e->getMessage(); + if ( $e instanceof Aws\Exception\AwsException ) { + $errorMessage = $e->getAwsErrorMessage(); + } $job_object->log( E_USER_ERROR, sprintf( __( 'S3 Service API: %s', 'backwpup' ), $errorMessage ), $e->getFile(),