Merge pull request #78 from ramindu90/master

mohanvive · web-flow · commit 51fcff8d1cf5 · 2019-12-12T10:32:41.000+05:30
Fix bugs including giving non existing paths as destination
diff --git a/component/pom.xml b/component/pom.xml
@@ -80,18 +80,19 @@
             <groupId>commons-io</groupId>
             <artifactId>commons-io</artifactId>
         </dependency>
+        <dependency>
+            <groupId>org.apache.commons</groupId>
+            <artifactId>commons-compress</artifactId>
+        </dependency>
         <dependency>
             <groupId>org.testng</groupId>
             <artifactId>testng</artifactId>
             <scope>test</scope>
         </dependency>
-        <dependency>
-            <groupId>org.apache.commons</groupId>
-            <artifactId>commons-compress</artifactId>
-        </dependency>
         <dependency>
             <groupId>io.siddhi.extension.execution.list</groupId>
             <artifactId>siddhi-execution-list</artifactId>
+            <scope>test</scope>
         </dependency>
     </dependencies>
     <profiles>
@@ -146,16 +147,20 @@
                             org.wso2.transport.remotefilesystem.*,
                             org.wso2.carbon.messaging.*,
                             org.apache.commons.vfs2.*,
+                            org.apache.commons.compress.*,
                             org.wso2.transport.file.*,
                             org.apache.commons.net.*,
-                            org.quartz.*
+                            org.quartz.*,
+                            io.siddhi.extension.util.*
                         </Private-Package>
                         <Export-Package>
                             io.siddhi.extension.io.file.*,
+                            io.siddhi.extension.execution.file.*,
                             !org.wso2.carbon.connector.framework.*,
                             !org.wso2.transport.remotefilesystem.*,
                             !org.wso2.carbon.messaging.*,
                             !org.apache.commons.vfs2.*,
+                            !org.apache.commons.compress.*,
                             !org.wso2.transport.file.*,
                             !org.apache.commons.net.*,
                             !org.quartz.*
diff --git a/component/src/main/java/io/siddhi/extension/execution/file/FileArchiveExtension.java b/component/src/main/java/io/siddhi/extension/execution/file/FileArchiveExtension.java
@@ -30,11 +30,14 @@
 import io.siddhi.core.query.processor.stream.function.StreamFunctionProcessor;
 import io.siddhi.core.util.config.ConfigReader;
 import io.siddhi.core.util.snapshot.state.StateFactory;
+import io.siddhi.extension.util.Utils;
 import io.siddhi.query.api.definition.AbstractDefinition;
 import io.siddhi.query.api.definition.Attribute;
 import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
 import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
 import org.apache.commons.io.IOUtils;
+import org.apache.commons.vfs2.FileObject;
+import org.apache.commons.vfs2.FileSystemException;
 import org.apache.log4j.Logger;
 
 import java.io.BufferedInputStream;
@@ -63,12 +66,14 @@
                 @Parameter(
                         name = "uri",
                         description = "Absolute path of the file or the directory",
-                        type = DataType.STRING
+                        type = DataType.STRING,
+                        dynamic = true
                 ),
                 @Parameter(
                         name = "destination.dir.uri",
                         description = "Absolute directory path of the the archived file.",
-                        type = DataType.STRING
+                        type = DataType.STRING,
+                        dynamic = true
                 ),
                 @Parameter(
                         name = "archive.type",
@@ -168,6 +173,9 @@ public ProcessingMode getProcessingMode() {
     protected Object[] process(Object[] data) {
         String uri = (String) data[0];
         String destinationDirUri = (String) data[1];
+        if (destinationDirUri.lastIndexOf(File.separator) != destinationDirUri.length() - 1) {
+            destinationDirUri = destinationDirUri + File.separator;
+        }
         boolean excludeSubdirectories = false;
         String regex = "";
         String archiveType = ZIP_FILE_EXTENSION;
@@ -183,20 +191,31 @@ protected Object[] process(Object[] data) {
         if (pattern == null) {
             pattern = Pattern.compile(regex);
         }
+        FileObject destinationDirUriObject = Utils.getFileObject(destinationDirUri);
+        try {
+            if (!destinationDirUriObject.exists() || !destinationDirUriObject.isFolder()) {
+                destinationDirUriObject.createFolder();
+            }
+        } catch (FileSystemException e) {
+            throw new SiddhiAppRuntimeException(
+                    "Exception occurred when creating the subdirectories for the destination directory  " +
+                            destinationDirUriObject.getName().getPath(), e);
+        }
+        File sourceFile = new File(uri);
+        String destinationFile = destinationDirUri + sourceFile.getName();
         if (archiveType.compareToIgnoreCase(ZIP_FILE_EXTENSION) == 0) {
-            File sourceFile = new File(uri);
             List<String> fileList = new ArrayList<>();
             generateFileList(uri, sourceFile, fileList, excludeSubdirectories);
             try {
-                zip(uri, destinationDirUri, fileList);
+                zip(uri, destinationFile, fileList);
             } catch (IOException e) {
                 throw new SiddhiAppRuntimeException("IOException occurred when archiving  " + uri, e);
             }
         } else {
             try {
                 if (archiveType.compareToIgnoreCase(TAR_FILE_EXTENSION) == 0) {
                     addToTarArchiveCompression(
-                            getTarArchiveOutputStream(destinationDirUri), new File(uri), uri);
+                            getTarArchiveOutputStream(destinationFile), sourceFile, uri);
                 } else {
                     throw new SiddhiAppRuntimeException("Unsupported archive type: " + archiveType);
                 }
@@ -233,7 +252,7 @@ private void zip(String sourceFileUri, String zipFile, List<String> fileList) th
         ZipOutputStream zos = null;
         FileOutputStream fos = null;
         try {
-            if (!zipFile.endsWith(TAR_FILE_EXTENSION)) {
+            if (!zipFile.endsWith(ZIP_FILE_EXTENSION)) {
                 zipFile = zipFile.concat("." + ZIP_FILE_EXTENSION);
             }
             fos = new FileOutputStream(zipFile);
diff --git a/component/src/main/java/io/siddhi/extension/execution/file/FileCopyExtension.java b/component/src/main/java/io/siddhi/extension/execution/file/FileCopyExtension.java
@@ -55,13 +55,15 @@
                 @Parameter(
                         name = "uri",
                         description = "Absolute path of the File or the directory.",
-                        type = DataType.STRING
+                        type = DataType.STRING,
+                        dynamic = true
                 ),
                 @Parameter(
                         name = "destination.dir.uri",
                         description = "Absolute path of the destination directory.\n" +
                                 "Note: Parent folder structure will be created if it does not exist.",
-                        type = DataType.STRING
+                        type = DataType.STRING,
+                        dynamic = true
                 ),
                 @Parameter(
                         name = "include.by.regexp",
diff --git a/component/src/main/java/io/siddhi/extension/execution/file/FileCreateExtension.java b/component/src/main/java/io/siddhi/extension/execution/file/FileCreateExtension.java
@@ -49,12 +49,14 @@
                 @Parameter(
                         name = "uri",
                         description = "Absolute file path which needs to be created.",
-                        type = DataType.STRING
+                        type = DataType.STRING,
+                        dynamic = true
                 ),
                 @Parameter(
                         name = "is.directory",
                         description = "This flag is used when creating file path is a directory",
-                        type = DataType.STRING
+                        type = DataType.STRING,
+                        dynamic = true
                 )
         },
         examples = {
diff --git a/component/src/main/java/io/siddhi/extension/execution/file/FileDeleteExtension.java b/component/src/main/java/io/siddhi/extension/execution/file/FileDeleteExtension.java
@@ -50,7 +50,8 @@
                 @Parameter(
                         name = "uri",
                         description = "Absolute path of the file or the directory to be deleted.",
-                        type = DataType.STRING
+                        type = DataType.STRING,
+                        dynamic = true
                 )
         },
         examples = {
diff --git a/component/src/main/java/io/siddhi/extension/execution/file/FileExistsExtension.java b/component/src/main/java/io/siddhi/extension/execution/file/FileExistsExtension.java
@@ -47,7 +47,8 @@
                 @Parameter(
                         name = "uri",
                         description = "File path to check for existence.",
-                        type = DataType.STRING
+                        type = DataType.STRING,
+                        dynamic = true
                 )
         },
         returnAttributes = {
diff --git a/component/src/main/java/io/siddhi/extension/execution/file/FileIsDirectoryExtension.java b/component/src/main/java/io/siddhi/extension/execution/file/FileIsDirectoryExtension.java
@@ -47,7 +47,8 @@
                 @Parameter(
                         name = "uri",
                         description = "The path to be checked for a directory.",
-                        type = DataType.STRING
+                        type = DataType.STRING,
+                        dynamic = true
                 )
         },
         returnAttributes = {
diff --git a/component/src/main/java/io/siddhi/extension/execution/file/FileIsFileExtension.java b/component/src/main/java/io/siddhi/extension/execution/file/FileIsFileExtension.java
@@ -47,7 +47,8 @@
                 @Parameter(
                         name = "file.path",
                         description = "The path to be checked for a file.",
-                        type = DataType.STRING
+                        type = DataType.STRING,
+                        dynamic = true
                 )
         },
         returnAttributes = {
diff --git a/component/src/main/java/io/siddhi/extension/execution/file/FileLastModifiedTimeExtension.java b/component/src/main/java/io/siddhi/extension/execution/file/FileLastModifiedTimeExtension.java
@@ -53,7 +53,8 @@
                 @Parameter(
                         name = "uri",
                         description = "File path to be checked for te last modified time.",
-                        type = DataType.STRING
+                        type = DataType.STRING,
+                        dynamic = true
                 ),
                 @Parameter(
                         name = "datetime.format",
diff --git a/component/src/main/java/io/siddhi/extension/execution/file/FileMoveExtension.java b/component/src/main/java/io/siddhi/extension/execution/file/FileMoveExtension.java
@@ -54,13 +54,15 @@
                 @Parameter(
                         name = "uri",
                         description = "Absolute file or directory path.",
-                        type = DataType.STRING
+                        type = DataType.STRING,
+                        dynamic = true
                 ),
                 @Parameter(
                         name = "destination.dir.uri",
                         description = "Absolute file path to the destination directory.\n" +
                                 "Note: Parent folder structure will be created if it does not exist.",
-                        type = DataType.STRING
+                        type = DataType.STRING,
+                        dynamic = true
                 ),
                 @Parameter(
                         name = "include.by.regexp",
diff --git a/component/src/main/java/io/siddhi/extension/execution/file/FileSearchExtension.java b/component/src/main/java/io/siddhi/extension/execution/file/FileSearchExtension.java
@@ -214,12 +214,12 @@ protected Object[] process(Object[] data) {
             throw new SiddhiAppRuntimeException("Exception occurred when getting the searching files in path " +
                     sourceFileUri, e);
         }
-        return fileList.toArray();
+        return new Object[]{fileList};
     }
 
     @Override
     protected Object[] process(Object data) {
-        return new Object[0];
+        return process(new Object[]{data});
     }
 
     /**
diff --git a/component/src/main/java/io/siddhi/extension/execution/file/FileSizeExtension.java b/component/src/main/java/io/siddhi/extension/execution/file/FileSizeExtension.java
@@ -53,7 +53,8 @@
                 @Parameter(
                         name = "uri",
                         description = "Absolute path to the file or directory to be checked for the size.",
-                        type = DataType.STRING
+                        type = DataType.STRING,
+                        dynamic = true
                 )
         },
         returnAttributes = {
diff --git a/component/src/main/java/io/siddhi/extension/execution/file/FileUnarchiveExtension.java b/component/src/main/java/io/siddhi/extension/execution/file/FileUnarchiveExtension.java
@@ -37,6 +37,7 @@
 import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
 import org.apache.commons.io.IOUtils;
 import org.apache.commons.vfs2.FileObject;
+import org.apache.commons.vfs2.FileSystemException;
 import org.apache.log4j.Logger;
 
 import java.io.File;
@@ -61,14 +62,16 @@
                 @Parameter(
                         name = "uri",
                         description = "Absolute path of the file to be decompressed in the format of zip or tar.",
-                        type = DataType.STRING
+                        type = DataType.STRING,
+                        dynamic = true
                 ),
                 @Parameter(
                         name = "destination.dir.uri",
                         description = "Absolute path of the destination directory.\n" +
                                 "Note: If the folder structure does not exist, it will be created.",
-                        type = DataType.STRING
-                ), // TODO: 11/28/19 find and replace folder with dir / directory
+                        type = DataType.STRING,
+                        dynamic = true
+                ),
                 @Parameter(
                         name = "exclude.root.dir",
                         description = "This flag excludes parent folder when extracting the content.",
@@ -142,13 +145,15 @@ protected Object[] process(Object[] data) {
                     destinationDirUri.concat(File.separator +
                             fileName.substring(0, fileName.lastIndexOf(sourceFileExtension) - 1));
         }
-        File destinationDirFile = new File(destinationDirUri);
+        FileObject destinationDirFile = Utils.getFileObject(destinationDirUri);
         // create output directory if it doesn't exist
-        if (!destinationDirFile.exists()) {
-            if (!destinationDirFile.mkdirs()) {
-                throw new SiddhiAppRuntimeException("Directory cannot be created to unzip the file: " +
-                        filePathUri);
+        try {
+            if (!destinationDirFile.exists() || !destinationDirFile.isFolder()) {
+                destinationDirFile.createFolder();
             }
+        } catch (FileSystemException e) {
+            throw new SiddhiAppRuntimeException("Directory cannot be created to unzip the file: " +
+                    filePathUri);
         }
         FileInputStream fis = null;
         //buffer for read and write data to file
@@ -163,32 +168,32 @@ protected Object[] process(Object[] data) {
                 ZipEntry ze = zis.getNextEntry();
                 while (ze != null) {
                     String fileName = ze.getName();
-                    filePath = destinationDirUri + File.separator + fileName;
-                    File newFile = new File(filePath);
-                    if (log.isDebugEnabled()) {
-                        log.debug("Decompressing: " + newFile.getAbsolutePath());
-                    }
-                    createParentDirectory(newFile, filePathUri);
-                    fos = new FileOutputStream(newFile);
-                    int len;
-                    while ((len = zis.read(buffer)) > 0) {
-                        fos.write(buffer, 0, len);
+                    if (!fileName.isEmpty()) {
+                        filePath = destinationDirUri + File.separator + fileName;
+                        File newFile = new File(filePath);
+                        if (log.isDebugEnabled()) {
+                            log.debug("Decompressing: " + newFile.getAbsolutePath());
+                        }
+                        createParentDirectory(newFile, filePathUri);
+                        fos = new FileOutputStream(newFile);
+                        int len;
+                        while ((len = zis.read(buffer)) > 0) {
+                            fos.write(buffer, 0, len);
+                        }
+                        fos.close();
                     }
-                    fos.close();
-                    //close this ZipEntry
+                    //Closes the current ZIP entry and positions the stream for reading the next entry
                     zis.closeEntry();
                     ze = zis.getNextEntry();
                 }
-                //close last ZipEntry
-                zis.closeEntry();
             } else if (sourceFileExtension.compareToIgnoreCase(Constant.TAR_FILE_EXTENSION) == 0) {
                 try (TarArchiveInputStream fin = new TarArchiveInputStream(new FileInputStream(filePathUri))) {
                     TarArchiveEntry entry;
                     while ((entry = fin.getNextTarEntry()) != null) {
                         if (entry.isDirectory()) {
                             continue;
                         }
-                        File curfile = new File(destinationDirFile, entry.getName());
+                        File curfile = new File(destinationDirFile.getName().getPath(), entry.getName());
                         createParentDirectory(curfile, filePathUri);
                         IOUtils.copy(fin, new FileOutputStream(curfile));
                     }
diff --git a/component/src/test/java/io/siddhi/extension/io/file/FileFunctionsTestCase.java b/component/src/test/java/io/siddhi/extension/io/file/FileFunctionsTestCase.java
diff --git a/component/src/test/resources/files/repo/function/archive/subFolder/test3.txt b/component/src/test/resources/files/repo/function/archive/subFolder/test3.txt
diff --git a/pom.xml b/pom.xml
