From 62dc4228c85e4faee258c0e6a5f72c9bfa39d877 Mon Sep 17 00:00:00 2001 From: Matthias Mohr Date: Tue, 30 Jul 2024 14:45:11 +0200 Subject: [PATCH] Implement draft for Google Drive Access --- package.json | 2 + src/api/worker/batchjob.js | 96 ++++++++++++++++++---------------- src/models/userstore.js | 7 +-- src/processes/utils/results.js | 3 +- src/utils/config.js | 2 + src/utils/gdrive.js | 96 ++++++++++++++++++++++++++++++++++ 6 files changed, 154 insertions(+), 52 deletions(-) create mode 100644 src/utils/gdrive.js diff --git a/package.json b/package.json index 530d089..f59943c 100644 --- a/package.json +++ b/package.json @@ -39,6 +39,7 @@ "dependencies": { "@google-cloud/storage": "^7.7.0", "@google/earthengine": "^0.1.385", + "@googleapis/drive": "^8.11.0", "@openeo/js-commons": "^1.4.1", "@openeo/js-processgraphs": "^1.3.0", "@seald-io/nedb": "^4.0.4", @@ -47,6 +48,7 @@ "check-disk-space": "^3.4.0", "epsg-index": "^2.0.0", "fs-extra": "^11.2.0", + "googleapis-common": "^7.2.0", "luxon": "^3.4.4", "proj4": "^2.10.0", "restify": "^11.1.0" diff --git a/src/api/worker/batchjob.js b/src/api/worker/batchjob.js index 04eddcf..6d6a344 100644 --- a/src/api/worker/batchjob.js +++ b/src/api/worker/batchjob.js @@ -3,6 +3,7 @@ import path from 'path'; import ProcessGraph from '../../processgraph/processgraph.js'; import GeeResults from '../../processes/utils/results.js'; import Utils from '../../utils/utils.js'; +import GDrive from '../../utils/gdrive.js'; const packageInfo = Utils.require('../../package.json'); export default async function run(config, storage, user, query) { @@ -34,22 +35,30 @@ export default async function run(config, storage, user, query) { const datacube = format.preprocess(GeeResults.BATCH, context, dc, logger); if (format.canExport()) { + // Ensure early that we have access to the Google Drive API + const drive = new GDrive(context.server(), user); + await drive.connect(); + // Start processing const tasks = await format.export(context.ee, dc, context.getResource()); storage.addTasks(job, tasks); context.startTaskMonitor(); - const filepath = await new Promise((resolve, reject) => { + const driveUrls = await new Promise((resolve, reject) => { setInterval(async () => { const updatedJob = await storage.getById(job._id, job.user_id); if (!updatedJob) { reject(new Error("Job was deleted")); } if (['canceled', 'error', 'finished'].includes(updatedJob.status)) { - // todo: resolve google drive URLs resolve(job.googleDriveResults); } }, 10000); }); - return { filepath, datacube }; + // Handle Google Drive specifics (permissions and public URLs) + const folderName = GDrive.getFolderName(job); + await drive.publishFoldersByName(folderName); + const files = await drive.getAssetsForFolder(folderName); + + return { files, datacube, links: driveUrls }; } else { const response = await format.retrieve(context.ee, dc); @@ -62,7 +71,7 @@ export default async function run(config, storage, user, query) { writer.on('error', reject); writer.on('close', resolve); }); - return { filepath, datacube }; + return { files: [filepath], datacube }; } }); @@ -70,13 +79,7 @@ export default async function run(config, storage, user, query) { const results = []; for (const task of computeTasks) { - const { filepath, datacube } = await task; - if (Array.isArray(filepath)) { - filepath.forEach(fp => results.push({ filepath: fp, datacube })); - } - else { - results.push({ filepath, datacube }); - } + results.push(await task); } const item = await createSTAC(storage, job, results); @@ -109,37 +112,12 @@ async function createSTAC(storage, job, results) { let startTime = null; let endTime = null; const extents = []; - for(const { filepath, datacube } of results) { - if (!filepath) { - continue; - } - - let asset; - let filename; - if (Utils.isUrl(filepath)) { - let url = new URL(filepath); - console.log(url); - filename = path.basename(url.pathname || url.hash.substring(1)); - asset = { - href: filepath, - roles: ["data"], -// type: Utils.extensionToMediaType(filepath), - title: filename - }; - } - else { - filename = path.basename(filepath); - const stat = await fse.stat(filepath); - asset = { - href: path.relative(folder, filepath), - roles: ["data"], - type: Utils.extensionToMediaType(filepath), - title: filename, - "file:size": stat.size, - created: stat.birthtime, - updated: stat.mtime - }; - } + for(const result of results) { + const files = result.files || []; + const datacube = result.datacube; + const baseAsset = { + roles: ["data"], + }; if (datacube.hasT()) { const t = datacube.dimT(); @@ -160,8 +138,8 @@ async function createSTAC(storage, job, results) { const extent = datacube.getSpatialExtent(); let wgs84Extent = extent; if (crs !== 4326) { - asset["proj:epsg"] = crs; - asset["proj:geometry"] = extent; + baseAsset["proj:epsg"] = crs; + baseAsset["proj:geometry"] = extent; wgs84Extent = Utils.projExtent(extent, 4326); } // Check the coordinates with a delta of 0.0001 or so @@ -171,8 +149,34 @@ async function createSTAC(storage, job, results) { } } - const params = datacube.getOutputFormatParameters(); - assets[filename] = Object.assign(asset, params.metadata); + for (const file of files) { + let asset; + let filename; + if (Utils.isUrl(file)) { + let url = new URL(file); + filename = path.basename(url.pathname || url.hash.substring(1)); + asset = { + href: file, + // type: Utils.extensionToMediaType(file), + title: filename + }; + } + else { + filename = path.basename(file); + const stat = await fse.stat(file); + asset = { + href: path.relative(folder, file), + type: Utils.extensionToMediaType(file), + title: filename, + "file:size": stat.size, + created: stat.birthtime, + updated: stat.mtime + }; + } + + const params = datacube.getOutputFormatParameters(); + assets[filename] = Object.assign(asset, baseAsset, params.metadata); + } } const item = { stac_version: packageInfo.stac_version, diff --git a/src/models/userstore.js b/src/models/userstore.js index cd3ef2e..146fc12 100644 --- a/src/models/userstore.js +++ b/src/models/userstore.js @@ -4,6 +4,7 @@ import Errors from '../utils/errors.js'; import crypto from "crypto"; import HttpUtils from '../utils/http.js'; import fse from 'fs-extra'; +import GDrive from '../utils/gdrive.js'; export default class UserStore { @@ -21,11 +22,7 @@ export default class UserStore { "openid", "email", "https://www.googleapis.com/auth/earthengine", - "https://www.googleapis.com/auth/drive.file", - // "https://www.googleapis.com/auth/drive", - // "https://www.googleapis.com/auth/cloud-platform", - // "https://www.googleapis.com/auth/devstorage.full_control" - ]; + ].concat(GDrive.SCOPES); } database() { diff --git a/src/processes/utils/results.js b/src/processes/utils/results.js index b9efbdd..7b2c8c4 100644 --- a/src/processes/utils/results.js +++ b/src/processes/utils/results.js @@ -1,3 +1,4 @@ +import GDrive from '../../utils/gdrive.js'; import Utils from '../../utils/utils.js'; import GeeProcessing from './processing.js'; import GeeTypes from './types.js'; @@ -76,7 +77,7 @@ const GeeResults = { const task = ee.batch.Export.image.toDrive({ image, description: job.title, - folder: 'gee-' + job._id, + folder: GDrive.getFolderName(job), fileNamePrefix: imageId, skipEmptyTiles: true, crs, diff --git a/src/utils/config.js b/src/utils/config.js index 2f1c4c2..746ceaa 100644 --- a/src/utils/config.js +++ b/src/utils/config.js @@ -29,6 +29,8 @@ export default class Config { certificate: null }; + // We need access to GEE + Drive + this.apiKey = null; this.serviceAccountCredentialsFile = null; this.googleAuthClients = []; diff --git a/src/utils/gdrive.js b/src/utils/gdrive.js new file mode 100644 index 0000000..788fc3a --- /dev/null +++ b/src/utils/gdrive.js @@ -0,0 +1,96 @@ +import drive from '@googleapis/drive'; +import { JWT } from 'googleapis-common'; +import Utils from './utils.js'; + +export default class GDrive { + + static SCOPES = [ + //"https://www.googleapis.com/auth/drive.file", + "https://www.googleapis.com/auth/drive" + ]; + + static getFolderName(job) { + return `gee-${job._id}`; + } + + constructor(context, user) { + this.drive = null; + this.context = context; + this.user = user; + } + + async connect() { + if (this.drive) { + return; + } + + let authType; + const options = { + version: 'v3' + }; + if (Utils.isGoogleUser(this.user._id)) { + authType = "user token"; + options.access_token = this.user.token; + options.auth = this.context.apiKey; + } + else if (this.context.eePrivateKey) { + authType = "private key"; + const client = this.context.eePrivateKey; + options.auth = new JWT(client.client_email, null, client.private_key, GDrive.SCOPES); + } + else { + throw new Error("No authentication method available, must have at least a private key configured."); + } + + this.drive = drive.drive(options); + // await this.drive.files.list({pageSize: 1}); + console.log(`Authenticated at Google Drive via ${authType}`); + } + + // Get the ID from URL + // https://drive.google.com/#folders/1rqL0rZqBCvNS9ZhgiJmPGN72y9ZfS3Ly + // => 1rqL0rZqBCvNS9ZhgiJmPGN72y9ZfS3Ly is the ID + getIdFromUrl(url) { + const parsed = new URL(url); + return parsed.hash.split('/').pop(); + } + + async publishFoldersByName(name) { + const res = await this.drive.files.list({ + q: `mimeType = 'application/vnd.google-apps.folder' and name = '${name}'`, + }); + const folders = res.data.files; + if (folders.length === 0) { + throw new Error(`Folder not found: ${name}`); + } + else { + console.log(folders); + const promises = folders.map(folder => this.publishFolder(folder.id)); + return Promise.all(promises); + } + } + + async publishFolder(id) { + if (Utils.isUrl(id)) { + id = this.getIdFromUrl(id); + } + return await this.drive.permissions.create({ + resource: { + 'type': 'anyone', + 'role': 'reader' + }, + fileId: id, + // fields: 'id', + }); + } + + async getAssetsForFolder(name) { + const res = await this.drive.files.list({ + pageSize: 1000, + q: `'${name}' in parents`, + }); + const files = res.data.files; + console.log(files); + return files; + } +}