diff --git a/README.md b/README.md
index a55a40e..b81e63d 100644
--- a/README.md
+++ b/README.md
@@ -2,6 +2,26 @@
Drop in replacement for the Node.js `fs` library backed by AWS S3.
+## Supported methods
+`@cyclic.sh/s3fs` supports the following `fs` methods operating on AWS S3:
+- writeFile / writeFileSync
+- readFile / readFileSync
+- exists / existsSync
+- rm / rmSync
+- stat / statSync
+- unlink / unlinkSync
+- readdir / readdirSync
+- mkdir / mkdirSync
+- rmdir / rmdirSync
+
+## Example Usage
+### Installation
+
+```
+npm install @cyclic.sh/s3fs
+```
+
+
Require in the same format as Node.js `fs`, specifying an S3 Bucket:
- Callbacks and Sync methods:
```js
@@ -12,42 +32,14 @@ Require in the same format as Node.js `fs`, specifying an S3 Bucket:
const fs = require('@cyclic.sh/s3fs/promises')(S3_BUCKET_NAME)
```
-## Supported methods
-`@cyclic.sh/s3fs` supports the following `fs` methods operating on AWS S3:
-- [x] fs.writeFile(filename, data, [options], callback)
- - [x] promise
- - [x] cb
- - [x] sync
-- [x] fs.readFile(filename, [options], callback)
- - [x] promise
- - [x] cb
- - [x] sync
-- [x] fs.exists(path, callback)
- - [x] promise
- - [x] cb
- - [x] sync
-- [x] fs.readdir(path, callback)
- - [x] promise
- - [x] cb
- - [x] sync
-- [x] fs.mkdir(path, [mode], callback)
- - [x] promise
- - [x] cb
- - [x] sync
-- [x] fs.stat(path, callback)
- - [x] promise
- - [x] cb
- - [x] sync
-- [ ] fs.rmdir(path, callback)
-- [ ] fs.rm(path, callback)
-- [ ] fs.unlink(path, callback)
-- [ ] fs.lstat(path, callback)
-- [ ] fs.createReadStream(path, [options])
-- [ ] fs.createWriteStream(path, [options])
-
-## Example Usage
### Authentication
-Authenticating the client can be done with one of two ways:
+
+Authenticating the client:
+- **cyclic.sh** -
+ - When deploying on cyclic.sh, credentials are already available in the environment
+ - The bucket name is also available under the `CYCLIC_BUCKET_NAME` variable
+ - read more: Cyclic Environment Variables
+- **Local Mode** - When no credentials are available - the client will fall back to using `fs` and the local filesystem with a warning.
- **Environment Variables** - the internal S3 client will use AWS credentials if set in the environment
```
AWS_REGION
@@ -62,9 +54,6 @@ Authenticating the client can be done with one of two ways:
credentials: {...}
})
```
-- **Local Mode** - When no credentials are available - the client will fall back to using `fs` and the local filesystem with a warning.
-
-
### Using Methods
The supported methods have the same API as Node.js `fs`:
- Sync
@@ -85,9 +74,4 @@ The supported methods have the same API as Node.js `fs`:
async function run(){
const json = JSON.parse(await fs.readFile('test/_read.json'))
}
- ```
-
-refer to fs, s3fs:
-
-- https://github.com/TooTallNate/s3fs
-- https://nodejs.org/docs/latest-v0.10.x/api/fs.html#fs_fs_mkdir_path_mode_callback
\ No newline at end of file
+ ```
\ No newline at end of file
diff --git a/src/CyclicS3FSPromises.js b/src/CyclicS3FSPromises.js
index cf3200f..b3ad367 100644
--- a/src/CyclicS3FSPromises.js
+++ b/src/CyclicS3FSPromises.js
@@ -4,7 +4,10 @@ const {
PutObjectCommand,
HeadObjectCommand,
ListObjectsCommand,
- ListObjectsV2Command
+ ListObjectsV2Command,
+ ListObjectVersionsCommand,
+ DeleteObjectCommand,
+ DeleteObjectsCommand,
} = require("@aws-sdk/client-s3");
const _path = require('path')
const {Stats} = require('fs')
@@ -68,9 +71,10 @@ class CyclicS3FSPromises{
}
async stat(fileName, data, options={}){
+ fileName = util.normalize_path(fileName)
const cmd = new HeadObjectCommand({
Bucket: this.bucket,
- Key: util.normalize_path(fileName)
+ Key: fileName
})
let result;
try{
@@ -98,7 +102,7 @@ class CyclicS3FSPromises{
}));
}catch(e){
if(e.name === 'NotFound'){
- throw new Error(`Error: ENOENT: no such file or directory, stat '${fileName}'`)
+ throw new Error(`ENOENT: no such file or directory, stat '${fileName}'`)
}else{
throw e
}
@@ -121,7 +125,7 @@ class CyclicS3FSPromises{
async readdir(path){
path = util.normalize_dir(path)
- const cmd = new ListObjectsCommand({
+ const cmd = new ListObjectsV2Command({
Bucket: this.bucket,
// StartAfter: path,
Prefix: path,
@@ -145,7 +149,7 @@ class CyclicS3FSPromises{
result = folders.concat(files).filter(r=>{return r.length})
}catch(e){
if(e.name === 'NotFound' || e.message === 'NotFound'){
- throw new Error(`Error: ENOENT: no such file or directory, scandir '${path}'`)
+ throw new Error(`ENOENT: no such file or directory, scandir '${path}'`)
}else{
throw e
}
@@ -153,6 +157,134 @@ class CyclicS3FSPromises{
return result
}
+ async rm(path){
+ try{
+ let f = await Promise.allSettled([
+ this.stat(path),
+ this.readdir(path)
+ ])
+
+ if(f[0].status == 'rejected' && f[1].status == 'fulfilled'){
+ throw new Error(`SystemError [ERR_FS_EISDIR]: Path is a directory: rm returned EISDIR (is a directory) ${path}`)
+ }
+ if(f[0].status == 'rejected' && f[1].status == 'rejected'){
+ throw f[0].reason
+ }
+
+ }catch(e){
+ throw e
+ }
+ path = util.normalize_path(path)
+ const cmd = new DeleteObjectCommand({
+ Bucket: this.bucket,
+ Key: path
+ })
+ try{
+ await this.s3.send(cmd)
+ }catch(e){
+ throw e
+ }
+
+ }
+
+ async rmdir(path){
+ try{
+ let contents = await this.readdir(path)
+ if(contents.length){
+ throw new Error(`ENOTEMPTY: directory not empty, rmdir '${path}'`)
+ }
+ }catch(e){
+ throw e
+ }
+
+ path = util.normalize_dir(path)
+ const cmd = new DeleteObjectCommand({
+ Bucket: this.bucket,
+ Key: path
+ })
+ try{
+ await this.s3.send(cmd)
+ }catch(e){
+ throw e
+ }
+ }
+
+ async unlink(path){
+ try{
+ let f = await Promise.allSettled([
+ this.stat(path),
+ this.readdir(path)
+ ])
+
+ if(f[0].status == 'rejected' && f[1].status == 'fulfilled'){
+ throw new Error(`EPERM: operation not permitted, unlink '${path}'`)
+ }
+ if(f[0].status == 'rejected' && f[1].status == 'rejected'){
+ throw f[0].reason
+ }
+
+ }catch(e){
+ throw e
+ }
+ path = util.normalize_path(path)
+ const cmd = new DeleteObjectCommand({
+ Bucket: this.bucket,
+ Key: path
+ })
+ try{
+ await this.s3.send(cmd)
+ }catch(e){
+ throw e
+ }
+ }
+
+
+ async deleteVersionMarkers(NextKeyMarker, list=[] ){
+ if (NextKeyMarker || list.length === 0) {
+ return await this.s3.send(new ListObjectVersionsCommand({
+ Bucket: this.bucket,
+ NextKeyMarker
+ })).then(async ({ DeleteMarkers, Versions, NextKeyMarker }) => {
+ if (DeleteMarkers && DeleteMarkers.length) {
+ await this.s3.send(new DeleteObjectsCommand({
+ Bucket: this.bucket,
+ Delete: {
+ Objects: DeleteMarkers.map((item) => ({
+ Key: item.Key,
+ VersionId: item.VersionId,
+ })),
+ },
+ }))
+
+ return await this.deleteVersionMarkers(NextKeyMarker, [
+ ...list,
+ ...DeleteMarkers.map((item) => item.Key),
+ ]);
+ }
+
+ if (Versions && Versions.length) {
+ await this.s3.send(new DeleteObjectsCommand({
+ Bucket: this.bucket,
+ Delete: {
+ Objects: Versions.map((item) => ({
+ Key: item.Key,
+ VersionId: item.VersionId,
+ })),
+ },
+ }))
+ return await this.deleteVersionMarkers(NextKeyMarker, [
+ ...list,
+ ...Versions.map((item) => item.Key),
+ ]);
+ }
+ return list;
+ });
+ }
+ return list;
+ };
+
+
+
}
diff --git a/src/index.js b/src/index.js
index 3a9a8e9..5971fd7 100644
--- a/src/index.js
+++ b/src/index.js
@@ -10,7 +10,6 @@ function makeCallback(cb) {
if (cb === undefined) {
return rethrow();
}
-
if (typeof cb !== 'function') {
throw new TypeError('callback must be a function');
}
@@ -98,6 +97,48 @@ class CyclicS3FS extends CyclicS3FSPromises {
})
}
+ rm(path, callback) {
+ callback = makeCallback(arguments[arguments.length - 1]);
+ new Promise(async (resolve,reject)=>{
+ try{
+ this.stat = super.stat
+ this.readdir = super.readdir
+ let res = await super.rm(...arguments)
+ return resolve(callback(null,res))
+ }catch(e){
+ return resolve(callback(e))
+ }
+ })
+ }
+
+ unlink(path, callback) {
+ callback = makeCallback(arguments[arguments.length - 1]);
+ new Promise(async (resolve,reject)=>{
+ try{
+ this.stat = super.stat
+ this.readdir = super.readdir
+ let res = await super.unlink(...arguments)
+ return resolve(callback(null,res))
+ }catch(e){
+ return resolve(callback(e))
+ }
+ })
+ }
+
+ rmdir(path, callback) {
+ callback = makeCallback(arguments[arguments.length - 1]);
+ new Promise(async (resolve,reject)=>{
+ try{
+ this.stat = super.stat
+ this.readdir = super.readdir
+ let res = await super.rmdir(...arguments)
+ return resolve(callback(null,res))
+ }catch(e){
+ return resolve(callback(e))
+ }
+ })
+ }
+
readFileSync(fileName) {
return sync_interface.runSync(this,'readFile',[fileName])
@@ -130,6 +171,18 @@ class CyclicS3FS extends CyclicS3FSPromises {
return sync_interface.runSync(this,'mkdir',[path])
}
+ rmSync(path) {
+ return sync_interface.runSync(this,'rm',[path])
+ }
+
+ unlinkSync(path) {
+ return sync_interface.runSync(this,'unlink',[path])
+ }
+
+ rmdirSync(path) {
+ return sync_interface.runSync(this,'rmdir',[path])
+ }
+
}
diff --git a/test/index.test.js b/test/index.test.js
index 03418ba..677d4bd 100644
--- a/test/index.test.js
+++ b/test/index.test.js
@@ -1,12 +1,28 @@
const path = require("path")
const BUCKET = process.env.BUCKET
-const { S3Client, PutObjectCommand } = require("@aws-sdk/client-s3");
+const {
+ S3Client,
+ PutObjectCommand,
+ } = require("@aws-sdk/client-s3");
const s3 = new S3Client({});
const s3fs = require("../src")
const s3fs_promises = require("../src/promises")
+afterAll(async () => {
+ let markers = await s3fs_promises(BUCKET).deleteVersionMarkers()
+ expect(markers.length > 10)
+
+ // need to run twice because need to delete the delete markers
+ markers = await s3fs_promises(BUCKET).deleteVersionMarkers()
+ expect(markers.length > 10)
+
+ // now should be empty
+ markers = await s3fs_promises(BUCKET).deleteVersionMarkers()
+ expect(markers.length == 0)
+})
+
beforeAll(async () => {
const fs = require("fs")
console.log('preparing test')
@@ -276,7 +292,235 @@ describe("Basic smoke tests", () => {
expect(contents).toEqual(['nested', 'file'])
})
+
+ test("rm() - promises", async () => {
+ const fs = s3fs_promises(BUCKET)
+ try{
+ await fs.rm('test/not_there.json')
+ }catch(e){
+ expect(e.message).toContain(`ENOENT: no such file or directory`)
+ }
+ await fs.writeFile('test/aaa.txt','asdfsdf')
+ await fs.rm('test/aaa.txt')
+
+ try{
+ await fs.stat('test/aaa.txt')
+ }catch(e){
+ expect(e.message).toContain(`ENOENT: no such file or directory`)
+ }
+ })
+ test("rmSync()", async () => {
+ const fs = s3fs(BUCKET)
+ try{
+ fs.rmSync('test/not_there.json')
+ }catch(e){
+ expect(e).toContain(`ENOENT: no such file or directory`)
+ }
+
+ fs.writeFileSync('test/aaa.txt','asdfsdf')
+ fs.rmSync('test/aaa.txt')
+
+ try{
+ await fs.statSync('test/aaa.txt')
+ }catch(e){
+ expect(e).toContain(`ENOENT: no such file or directory`)
+ }
+ })
+
+
+ test("rm() - callback", async () => {
+
+ const fs = s3fs(BUCKET)
+ await new Promise((resolve,reject)=>{
+ fs.rm('test/not_there.txt', (error,data) =>{
+ expect(error.message).toContain(`ENOENT: no such file or directory`)
+ resolve()
+ })
+ })
+ fs.writeFileSync('test/aaa.txt','asdfsdf')
+
+ await new Promise((resolve,reject)=>{
+ fs.rm('test/aaa.txt',(error,data)=>{
+ expect(error).toEqual(null)
+ resolve()
+ })
+ })
+ })
+
+ test("rmSync(directory)", async () => {
+ const fs = s3fs(BUCKET)
+ fs.mkdirSync('/dir')
+
+ try{
+ fs.rmSync('/dir')
+ }catch(e){
+ expect(e).toContain(`[ERR_FS_EISDIR]: Path is a directory: rm returned EISDIR (is a directory)`)
+ }
+
+ })
+
+ test("unlink() - promises", async () => {
+ const fs = s3fs_promises(BUCKET)
+ try{
+ await fs.rm('test/not_there.json')
+ }catch(e){
+ expect(e.message).toContain(`ENOENT: no such file or directory`)
+ }
+ await fs.writeFile('test/aaa.txt','asdfsdf')
+ await fs.unlink('test/aaa.txt')
+
+ try{
+ await fs.stat('test/aaa.txt')
+ }catch(e){
+ expect(e.message).toContain(`ENOENT: no such file or directory`)
+ }
+ })
+
+ test("unlinkSync()", async () => {
+ const fs = s3fs(BUCKET)
+ try{
+ fs.unlinkSync('test/not_there.json')
+ }catch(e){
+ expect(e).toContain(`ENOENT: no such file or directory`)
+ }
+
+ fs.writeFileSync('test/aaa.txt','asdfsdf')
+ fs.unlinkSync('test/aaa.txt')
+
+ try{
+ await fs.statSync('test/aaa.txt')
+ }catch(e){
+ expect(e).toContain(`ENOENT: no such file or directory`)
+ }
+ })
+
+
+ test("unlink() - callback", async () => {
+
+ const fs = s3fs(BUCKET)
+ await new Promise((resolve,reject)=>{
+ fs.unlink('test/not_there.txt', (error,data) =>{
+ expect(error.message).toContain(`ENOENT: no such file or directory`)
+ resolve()
+ })
+ })
+ fs.writeFileSync('test/aaa.txt','asdfsdf')
+
+ await new Promise((resolve,reject)=>{
+ fs.unlink('test/aaa.txt',(error,data)=>{
+ expect(error).toEqual(null)
+ resolve()
+ })
+ })
+ })
+
+
+ test("unlinkSync(directory)", async () => {
+ const fs = s3fs(BUCKET)
+ fs.mkdirSync('/dir')
+
+ try{
+ fs.unlinkSync('/dir')
+ }catch(e){
+ expect(e).toContain(`EPERM: operation not permitted, unlink`)
+ }
+
+ })
+
+
+
+ test("rmdir() - promises", async () => {
+ const fs = s3fs_promises(BUCKET)
+ try{
+ await fs.rmdir('not_there')
+ }catch(e){
+ expect(e.message).toContain(`ENOENT: no such file or directory`)
+ }
+ let dir_name = `dir_${Date.now()}`
+ await fs.mkdir(dir_name)
+ await fs.rmdir(dir_name)
+
+ try{
+ await fs.readdir(dir_name)
+ }catch(e){
+ expect(e.message).toContain(`ENOENT: no such file or directory`)
+ }
+ })
+
+ test("rmdirSync()", async () => {
+ const fs = s3fs(BUCKET)
+ try{
+ fs.rmdirSync('not_there')
+ }catch(e){
+ expect(e).toContain(`ENOENT: no such file or directory`)
+ }
+ let dir_name = `dir_${Date.now()}`
+ fs.mkdirSync(dir_name)
+ fs.rmdirSync(dir_name)
+
+ try{
+ fs.readdirSync(dir_name)
+ }catch(e){
+ expect(e).toContain(`ENOENT: no such file or directory`)
+ }
+ })
+
+
+ test("rmdir() - callback", async () => {
+
+ const fs = s3fs(BUCKET)
+ await new Promise((resolve,reject)=>{
+ fs.rmdir('not_there', (error,data) =>{
+ expect(error.message).toContain(`ENOENT: no such file or directory`)
+ resolve()
+ })
+ })
+ let dir_name = `/dir_${Date.now()}`
+
+ fs.mkdirSync(dir_name)
+ await new Promise((resolve,reject)=>{
+ fs.rmdir(dir_name,(error,data)=>{
+ expect(error).toEqual(null)
+ resolve()
+ })
+ })
+
+ await new Promise((resolve,reject)=>{
+ fs.rmdir(dir_name, (error,data) =>{
+ expect(error.message).toContain(`ENOENT: no such file or directory`)
+ resolve()
+ })
+ })
+
+ })
+
+
+
+ test("rmdirSync() - not empty", async () => {
+ const fs = s3fs(BUCKET)
+ let dir_name = `/nested/dir_${Date.now()}`
+ fs.mkdirSync(dir_name)
+ try{
+ fs.rmdirSync('/nested')
+ }catch(e){
+ expect(e).toContain(`ENOTEMPTY: directory not empty, rmdir`)
+ }
+
+ })
+
+ test("empty_bucket", async () => {
+ const fs = s3fs(BUCKET)
+ let dir_name = `/nested/dir_${Date.now()}`
+ fs.mkdirSync(dir_name)
+ try{
+ fs.rmdirSync('/nested')
+ }catch(e){
+ expect(e).toContain(`ENOTEMPTY: directory not empty, rmdir`)
+ }
+
+ })
+