Merge pull request #7 from cyclic-software/dir-methods

Dir methods

README.md

@@ -26,18 +26,24 @@ Require in the same format as Node.js `fs`, specifying an S3 Bucket:
   - [x] promise
   - [x] cb
   - [x] sync
-- [ ] fs.readdir(path, callback)
-- [ ] fs.mkdir(path, [mode], callback)
-- [ ] fs.rmdir(path, callback)
+- [x] fs.readdir(path, callback)
+  - [x] promise
+  - [x] cb
+  - [x] sync
+- [x] fs.mkdir(path, [mode], callback)
+  - [x] promise
+  - [x] cb
+  - [x] sync
 - [x] fs.stat(path, callback)
   - [x] promise
   - [x] cb
   - [x] sync
+- [ ] fs.rmdir(path, callback)
+- [ ] fs.rm(path, callback)
+- [ ] fs.unlink(path, callback)
 - [ ] fs.lstat(path, callback)
 - [ ] fs.createReadStream(path, [options])
 - [ ] fs.createWriteStream(path, [options])
-- [ ] fs.unlink(path, callback)
-- [ ] fs.rm(path, callback)
 
 ## Example Usage
 ### Authentication

src/CyclicS3FSPromises.js

@@ -3,9 +3,12 @@ const {
   GetObjectCommand, 
   PutObjectCommand,
   HeadObjectCommand,
+  ListObjectsCommand,
+  ListObjectsV2Command
 } = require("@aws-sdk/client-s3");
-
+const _path = require('path')
 const {Stats} = require('fs')
+const util = require('./util')
 const sync_interface = require('./sync_interface');
 function streamToBuffer(stream) {
   return new Promise((resolve, reject) => {
@@ -26,7 +29,7 @@ class CyclicS3FSPromises{
   async readFile(fileName ,options){
     const cmd = new GetObjectCommand({
       Bucket: this.bucket,
-      Key: fileName,
+      Key: util.normalize_path(fileName),
       })
 
     let obj = await this.s3.send(cmd)
@@ -37,7 +40,7 @@ class CyclicS3FSPromises{
   async writeFile(fileName, data, options={}){
     const cmd = new PutObjectCommand({
         Bucket: this.bucket,
-        Key: fileName,
+        Key: util.normalize_path(fileName), 
         Body: data
     })
     await this.s3.send(cmd)
@@ -46,7 +49,7 @@ class CyclicS3FSPromises{
   async exists(fileName, data, options={}){
     const cmd = new HeadObjectCommand({
         Bucket: this.bucket,
-        Key: fileName
+        Key: util.normalize_path(fileName)
     })
     let exists
     try{
@@ -67,7 +70,7 @@ class CyclicS3FSPromises{
   async stat(fileName, data, options={}){
     const cmd = new HeadObjectCommand({
         Bucket: this.bucket,
-        Key: fileName
+        Key: util.normalize_path(fileName)
     })
     let result;
     try{
@@ -102,6 +105,53 @@ class CyclicS3FSPromises{
     }
     return result
   }
+
+  async mkdir(path){
+    path = util.normalize_dir(path)
+    const cmd = new PutObjectCommand({
+        Bucket: this.bucket,
+        Key: path,
+    })
+    try{
+        await this.s3.send(cmd)
+    }catch(e){
+        throw e
+    }
+  }
+
+  async readdir(path){
+    path = util.normalize_dir(path)
+    const cmd = new ListObjectsCommand({
+        Bucket: this.bucket,
+        // StartAfter: path,
+        Prefix: path,
+        // Delimiter: '/' 
+        Delimiter: _path.sep 
+    })
+    let result;
+    try{
+        result = await this.s3.send(cmd)
+        if(!result.Contents && !result.CommonPrefixes){
+            throw new Error('NotFound')
+        }
+        let trailing_sep = new RegExp(`${_path.sep}$`)
+
+        let folders = (result.CommonPrefixes || []).map(r=>{
+            return r.Prefix.replace(path, '').replace(trailing_sep, "");
+        })
+        let files = (result.Contents || []).map(r=>{
+            return r.Key.replace(path, '')
+        })
+        result = folders.concat(files).filter(r=>{return r.length})
+    }catch(e){
+      if(e.name === 'NotFound' || e.message === 'NotFound'){
+        throw new Error(`Error: ENOENT: no such file or directory, scandir '${path}'`)
+      }else{
+        throw e
+      }
+    }
+    return result
+  }
 
 }
 

src/index.js

@@ -74,6 +74,30 @@ class CyclicS3FS extends CyclicS3FSPromises {
     })
   }
 
+  readdir(path, callback) {
+    callback = makeCallback(arguments[arguments.length - 1]);
+    new Promise(async (resolve,reject)=>{
+      try{
+        let res = await super.readdir(...arguments)
+        return resolve(callback(null,res))
+      }catch(e){
+        return resolve(callback(e))
+      }
+    })
+  }
+
+  mkdir(path, callback) {
+    callback = makeCallback(arguments[arguments.length - 1]);
+    new Promise(async (resolve,reject)=>{
+      try{
+        let res = await super.mkdir(...arguments)
+        return resolve(callback(null,res))
+      }catch(e){
+        return resolve(callback(e))
+      }
+    })
+  }
+
 
   readFileSync(fileName) {
     return sync_interface.runSync(this,'readFile',[fileName])
@@ -96,6 +120,16 @@ class CyclicS3FS extends CyclicS3FSPromises {
     return res
   }
 
+  readdirSync(path) {
+    let res = sync_interface.runSync(this,'readdir',[path])
+    res = v8.deserialize(res)
+    return res
+  }
+
+  mkdirSync(path) {
+    return sync_interface.runSync(this,'mkdir',[path])
+  }
+
 }
 
 

src/sync_interface.js

@@ -51,7 +51,7 @@ const run = async function(bucket, config, method, args){
     const fs = new CyclicS3FSPromises(bucket, config)
     let result = await fs[method](...args)
     if(typeof result !== 'undefined'){
-        if(['stat','exists'].includes(method)){
+        if(['stat','exists','readdir'].includes(method)){
             result = v8.serialize(result)
         }
         process.stdout.write(result);

src/util.js

@@ -0,0 +1,22 @@
+const path = require('path')
+const normalize_path = function (p) {
+    p = path.normalize(p);
+    p = path.resolve(process.cwd(), p)
+    // p = path.relative(require.main.path, p)
+    p = p.replace(new RegExp(`^${path.sep}+`, 'g'), '');
+    // p = `${p}`
+    // p = `s3fs:${p}`
+    return p
+}
+
+const normalize_dir = function(p){
+    p = normalize_path(p)
+    p = `${p}${path.sep}`
+    return p
+}
+
+
+module.exports = {
+    normalize_path,
+    normalize_dir
+}

test/index.test.js

@@ -1,4 +1,4 @@
-const fs = require("fs")
+
 const path = require("path")
 const BUCKET = process.env.BUCKET
 const { S3Client, PutObjectCommand } = require("@aws-sdk/client-s3");
@@ -8,18 +8,19 @@ const s3fs = require("../src")
 const s3fs_promises = require("../src/promises")
 
 beforeAll(async () => {
+  const fs = require("fs")
   console.log('preparing test')
   try{
 
     await s3.send(new PutObjectCommand({
       Bucket: BUCKET,
-      Key: 'test/_read.json',
+      Key: `${path.resolve(__dirname,'./_read.json').replace(new RegExp(`^${path.sep}+`, 'g'), '')}`,
       Body: fs.readFileSync(path.resolve(__dirname,'./_read.json')),
       ContentType: 'application/json'
     }))
     await s3.send(new PutObjectCommand({
       Bucket: BUCKET,
-      Key: 'test/_read.jpeg',
+      Key: `${path.resolve(__dirname,'./_read.jpeg').replace(new RegExp(`^${path.sep}+`, 'g'), '')}`,
       Body: fs.readFileSync(path.resolve(__dirname,'./_read.jpeg')),
       ContentType: 'image/jpeg'
     }))
@@ -205,6 +206,78 @@ describe("Basic smoke tests", () => {
 
   })
 
+
+  test("mkdir(), readdir() - promises", async () => {
+    const fs = s3fs_promises(BUCKET)
+    let dir_name = `dir_${Date.now()}`
+    try{
+      let d = await fs.readdir(dir_name)
+    }catch(e){
+      expect(e.message).toContain(`ENOENT: no such file or directory`)
+    }
+    await fs.mkdir(dir_name)
+
+    let d = await fs.readdir(dir_name)
+    expect(d).toEqual([])
+
+  })
+
+
+  test("mkdir(), readdir() - callback", async () => {
+    const fs = s3fs(BUCKET)
+    await new Promise((resolve,reject)=>{
+      let dir_name = `dir_${Date.now()}`
+      fs.mkdir(dir_name, ()=>{
+        fs.readdir(dir_name, (error, result)=>{
+          expect(result).toEqual([])
+          resolve()
+        })
+      })
+    })
+
+    await new Promise((resolve,reject)=>{
+      let dir_name = `dir_not_there_${Date.now()}`
+      fs.readdir(dir_name, (error, result)=>{
+          expect(error.message).toContain(`ENOENT: no such file or directory`)
+          resolve()
+      })
+    })
+
+  })
+
+
+  test("readdirSync(), mkdirSync()", async () => {
+    const fs = s3fs(BUCKET)
+    let dir_name = `dir_${Date.now()}`
+    try{
+      fs.readdirSync(dir_name)
+    }catch(e){
+      expect(e).toContain(`ENOENT: no such file or directory`)
+    }
+    fs.mkdirSync(dir_name)
+    let contents = fs.readdirSync(dir_name)
+    expect(contents).toEqual([])
+
+  })
 
 
+  test("readdir(), mkdir() - nested", async () => {
+    const fs = s3fs_promises(BUCKET)
+    let dir_name = `/dir_nested_${Date.now()}`
+
+    await fs.mkdir(`${dir_name}/nested`)
+
+    contents = await fs.readdir(dir_name)
+    expect(contents).toEqual(['nested'])
+
+    await fs.writeFile(`${dir_name}/file`,Date.now().toString())
+
+    contents = await fs.readdir(dir_name)
+    expect(contents).toEqual(['nested', 'file'])
+
+  })
+
+
+
+
 })