Merge pull request #30 from daita/0.4.x

0.4.2: - extract and index files from trash - index remove orphans
nextcloud · Oct 4, 2016 · b75c9a1 · b75c9a1
2 parents 9967b94 + c645faa
commit b75c9a1
Show file tree

Hide file tree

Showing 12 changed files with 287 additions and 89 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,3 +1,16 @@
+**v0.4.2**
+- extract and index files from trash
+- index remove orphans
+
+
+**v0.4.1**
+- Force index unlock after 24h
+- maximum size on extracted file
+- new field nextant_path
+- documents with its file deleted are removed
+- bugfixes
+
+
 **v0.4.0**
 - index safe guard
 - new options on nextant:index

diff --git a/appinfo/info.xml b/appinfo/info.xml
@@ -13,7 +13,7 @@ _Recognized file format: text, rtf, pdf, html, openoffice, office, ..._
 
 
 	</description>
-	<version>0.4.1</version>
+	<version>0.4.2</version>
 	<licence>agpl</licence>
 	<author>Maxence Lange</author>
 	<types>

diff --git a/appinfo/register_command.php b/appinfo/register_command.php
@@ -27,7 +27,7 @@
 $app = new \OCA\Nextant\AppInfo\Application();
 $c = $app->getContainer();
 
-$application->add(new OCA\Nextant\Command\Check($c->query('SolrService'), $c->query('SolrAdminService')));
+$application->add(new OCA\Nextant\Command\Check($c->query('SolrService'), $c->query('SolrToolsService'), $c->query('SolrAdminService')));
 $application->add(new OCA\Nextant\Command\Clear($c->query('ConfigService'), $c->query('SolrService'), $c->query('SolrAdminService')));
 $application->add(new OCA\Nextant\Command\Scan(OC::$server->getUserManager(), $c->query('UserFolder'), $c->query('SolrService'), $c->query('FileService')));
 $application->add(new OCA\Nextant\Command\Index(OC::$server->getUserManager(), $c->query('RootFolder'), $c->query('SolrService'), $c->query('SolrToolsService'), $c->query('ConfigService'), $c->query('FileService'), $c->query('MiscService')));

diff --git a/lib/Command/Check.php b/lib/Command/Check.php
@@ -37,12 +37,15 @@ class Check extends Base
 
     private $solrService;
 
+    private $solrTools;
+
     private $solrAdmin;
 
-    public function __construct($solrService, $solrAdmin)
+    public function __construct($solrService, $solrTools, $solrAdmin)
     {
         parent::__construct();
         $this->solrService = $solrService;
+        $this->solrTools = $solrTools;
         $this->solrAdmin = $solrAdmin;
     }
 
@@ -53,11 +56,10 @@ protected function configure()
     }
 
     protected function execute(InputInterface $input, OutputInterface $output)
-    {       
-        if (!$this->solrService->configured())
-        {
+    {
+        if (! $this->solrService->configured()) {
             $output->writeln('Nextant is not yet configured');
-            return;            
+            return;
         }
 
         $this->solrService->setOutput($output);
@@ -75,7 +77,7 @@ protected function execute(InputInterface $input, OutputInterface $output)
             return false;
         }
 
-        $output->writeln('Your solr contains ' . $this->solrAdmin->count() . ' documents');
+        $output->writeln('Your solr contains ' . $this->solrTools->count() . ' documents');
     }
 }
 

diff --git a/lib/Command/Index.php b/lib/Command/Index.php
@@ -26,7 +26,6 @@
  */
 namespace OCA\Nextant\Command;
 
-use \OCA\Nextant\Service\SolrToolsService;
 use \OCA\Nextant\Service\FileService;
 use OC\Core\Command\Base;
 use Symfony\Component\Console\Input\InputArgument;
@@ -108,10 +107,12 @@ protected function execute(InputInterface $input, OutputInterface $output)
 
         $this->configService->setAppValue('solr_lock', time());
 
+        $documentIds = array();
+
         //
         // extract files
         $output->writeln('');
-        $output->writeln('* Extracting new files to Solr:');
+        $output->writeln('* Extracting files to Solr:');
         $output->writeln('');
 
         $users = $this->userManager->search('');
@@ -130,17 +131,22 @@ protected function execute(InputInterface $input, OutputInterface $output)
             $this->solrService->setOwner($userId);
 
             $this->miscService->debug('Init Extracting new files for user ' . $userId);
-            $result = $this->browseUserDirectory($userId, $output);
+            $result_files = $this->browseUserDirectory($output, $userId, '/files', array(), $docIds_files);
 
-            if ($result['total'] > 0) {
-
+            $output->writeln('');
+            $result_trash = $this->browseUserDirectory($output, $userId, '/files_trashbin', array(
+                'deleted' => true
+            ), $docIds_trash);
+
+            if (($result_files['total'] + $result_trash['total']) > 0) {
+                $documentIds = array_merge($documentIds, $docIds_files, $docIds_trash);
                 array_push($extractedDocuments, array(
                     'userid' => $userId,
-                    'files' => $result['files']
+                    'files' => array_merge($result_files['files'], $result_trash['files'])
                 ));
 
-                $extractedTotal += sizeof($result['files']);
-                $processedTotal += $result['processed'];
+                $extractedTotal += sizeof($result_files['files']) + sizeof($result_trash['files']);
+                $processedTotal += $result_files['processed'] + $result_trash['processed'];
             }
 
             $output->writeln('');
@@ -152,9 +158,12 @@ protected function execute(InputInterface $input, OutputInterface $output)
         // update Documents
         $output->writeln('');
         $output->writeln('* Updating documents status');
+        $output->writeln('');
+
         $noFailure = true;
         $usersTotal = sizeof($extractedDocuments);
         $usersCurrent = 0;
+        $processedFile = 0;
         foreach ($extractedDocuments as $doc) {
             $usersCurrent ++;
 
@@ -164,13 +173,27 @@ protected function execute(InputInterface $input, OutputInterface $output)
 
             $this->miscService->debug('Init Updating documents for user ' . $userId);
 
-            if (! $this->updateUserDocuments($userId, $fileIds, $output))
+            if (! $this->updateUserDocuments($userId, $fileIds, $output, $updateDocs))
                 $noFailure = false;
 
+            $processedFile += $updateDocs;
             $output->writeln('');
         }
 
+        $output->writeln('       ' . $processedFile . ' file(s) updated');
+
+        // $output->writeln(' - ' . $deleted . ' documents removed');
+
         Filesystem::tearDown();
+
+        //
+        // removing orphan
+        $output->writeln('');
+        $output->writeln('* Removing orphan documents');
+        $output->writeln('');
+
+        $this->removeOrphans($output, $documentIds);
+
         $this->configService->needIndex(false);
 
         if ($noFailure)
@@ -179,42 +202,45 @@ protected function execute(InputInterface $input, OutputInterface $output)
             $this->configService->needIndex(true);
 
         $this->configService->setAppValue('solr_lock', '0');
+
+        $output->writeln('');
     }
 
-    private function browseUserDirectory($userId, $output)
+    private function browseUserDirectory($output, $userId, $dir, $options, &$docIds)
     {
         Filesystem::tearDown();
         Filesystem::init($userId, '');
         $this->fileService->setView(Filesystem::getView());
         $this->miscService->debug('(' . $userId . ') - Init Filesystem');
 
-        $userFolder = FileService::getUserFolder($this->rootFolder, $userId, '/files');
+        $userFolder = FileService::getUserFolder($this->rootFolder, $userId, $dir);
         if ($userFolder != null && $userFolder) {
             $folder = $userFolder->get('/');
 
-            $this->miscService->debug('(' . $userId . ') - found root folder');
+            $this->miscService->debug('(' . $userId . '/' . $dir . ') - found root folder');
             $files = $folder->search('');
         } else
             $files = array();
 
         $this->miscService->debug('(' . $userId . ') - found ' . sizeof($files) . ' files');
 
         $progress = new ProgressBar($output, sizeof($files));
-        $progress->setMessage('<info>' . $userId . '</info>: ');
+        $progress->setMessage('<info>' . $userId . '</info>' . $dir . ': ');
         $progress->setMessage('', 'jvm');
         $progress->setMessage('[preparing]', 'infos');
-        $progress->setFormat(" %message:-30s%%current:5s%/%max:5s% [%bar%] %percent:3s%% \n    %infos:1s% %jvm:-30s%      ");
+        $progress->setFormat(" %message:-38s%%current:5s%/%max:5s% [%bar%] %percent:3s%% \n    %infos:1s% %jvm:-30s%      ");
         $progress->start();
 
         if (sizeof($files) > 10)
             sleep(5);
 
         $filesProcessed = 0;
         $fileIds = array();
+        $docIds = array();
         $lastProgressTick = 0;
         foreach ($files as $file) {
 
-            $this->miscService->debug('(' . $userId . ') - scaning file #' . $file->getId() . ' (' . $file->getMimeType() . ') ' . $file->getPath());
+            $this->miscService->debug('(' . $userId . ') - scanning file #' . $file->getId() . ' (' . $file->getMimeType() . ') ' . $file->getPath());
 
             if ($this->hasBeenInterrupted()) {
                 $this->configService->setAppValue('solr_lock', '0');
@@ -233,10 +259,11 @@ private function browseUserDirectory($userId, $output)
 
                 $forceExtract = false;
                 $status = 0;
-
                 if ($this->fileService->addFileFromPath($file->getPath(), $forceExtract, $status)) {
+                    array_push($docIds, (int) $file->getId());
                     array_push($fileIds, array(
                         'fileid' => $file->getId(),
+                        'options' => $options,
                         'path' => $file->getPath()
                     ));
                     $filesProcessed += $status;
@@ -261,7 +288,7 @@ private function browseUserDirectory($userId, $output)
         );
     }
 
-    private function updateUserDocuments($userId, $fileIds, $output)
+    private function updateUserDocuments($userId, $fileIds, $output, &$processedfile)
     {
         Filesystem::tearDown();
         Filesystem::init($userId, '');
@@ -271,7 +298,7 @@ private function updateUserDocuments($userId, $fileIds, $output)
         // $cycle = array_chunk($fileIds, 5);
 
         $progress = new ProgressBar($output, sizeof($fileIds));
-        $progress->setFormat(" %message:-30s%%current:5s%/%max:5s%  [%bar%] %percent:3s%% \n    %infos:1s% %jvm:-30s% %failures:1s%     ");
+        $progress->setFormat(" %message:-38s%%current:5s%/%max:5s%  [%bar%] %percent:3s%% \n    %infos:1s% %jvm:-30s% %failures:1s%     ");
         $progress->setMessage('<info>' . $userId . '</info>: ');
         $progress->setMessage('', 'jvm');
         $progress->setMessage('', 'failures');
@@ -283,27 +310,32 @@ private function updateUserDocuments($userId, $fileIds, $output)
         sleep(5);
         $i = 0;
         $lastProgressTick = 0;
-        $failure = 0;
-        foreach ($fileIds as $file) {
+        $failureIds = array();
+        $processedfile = 0;
+        while ($file = array_shift($fileIds)) {
+
             if ($this->hasBeenInterrupted()) {
                 $this->configService->setAppValue('solr_lock', '0');
                 throw new \Exception('ctrl-c');
             }
 
-            $result = $this->fileService->updateFiles(array(
+            $count = $this->fileService->updateFiles(array(
                 $file
-            ));
-            $progress->setMessage('failure(s): ' . $failure, 'failures');
+            ), $file['options']);
 
-            if ($result)
-                $this->miscService->debug('' . $userId . ' update done');
-            else {
-                $failure ++;
-                $progress->setMessage('failure(s): ' . $failure, 'failures');
+            // $progress->setMessage('failure(s): ' . $failure, 'failures');
+
+            if ($count === false) {
+                array_push($failureIds, $file);
+                $progress->setMessage('failure(s): ' . sizeof($failureIds), 'failures');
                 $this->miscService->debug('' . $userId . ' update failed');
+            } else {
+                $processedfile += $count;
+                $this->miscService->debug('' . $userId . ' update done');
             }
 
             $progress->setMessage('[updating] -', 'infos');
+
             if ((time() - self::REFRESH_INFO_SYSTEM) > $lastProgressTick) {
                 $infoSystem = $this->solrTools->getInfoSystem();
                 $progress->setMessage($infoSystem->jvm->memory->used, 'jvm');
@@ -317,17 +349,72 @@ private function updateUserDocuments($userId, $fileIds, $output)
 
             $i ++;
 
-            // let's take a break every 1000 files
+            // let's take a break every 500 files
             if (($i % 500) == 0) {
                 $progress->setMessage('[standby] -', 'infos');
                 $progress->display();
                 sleep(2);
             }
         }
 
+        $progress->setMessage('', 'jvm');
+        $progress->setMessage('', 'infos');
+        $progress->finish();
+
+        return (sizeof($failureIds) == 0);
+    }
+
+    private function removeOrphans($output, $fileIds)
+    {
+        $progress = new ProgressBar($output, $this->solrTools->count());
+
+        $progress->setMessage('<info>spoting orphans</info>:');
+        $progress->setFormat(" %message:-51s%[%bar%] %percent:3s%%");
+        $progress->start();
+
+        $deleting = array();
+        $page = 0;
+        while (true) {
+
+            if ($this->hasBeenInterrupted()) {
+                $this->configService->setAppValue('solr_lock', '0');
+                throw new \Exception('ctrl-c');
+            }
+
+            $ids = $this->solrTools->getAll($page, $lastPage, $error);
+            if (! $ids)
+                return false;
+
+            foreach ($ids as $id) {
+
+                if (! in_array($id, $fileIds) && (! in_array($id, $deleting)))
+                    array_push($deleting, $id);
+
+                $progress->advance();
+            }
+
+            if ($lastPage)
+                break;
+            $page ++;
+        }
+
         $progress->finish();
+        $output->writeln('');
 
-        return ($failure == 0);
+        if (sizeof($deleting) > 0) {
+            $progress = new ProgressBar($output, sizeof($deleting));
+            $progress->setMessage('<info>removing orphans</info>:');
+            $progress->setFormat(" %message:-38s%%current:5s%/%max:5s%  [%bar%] %percent:3s%%");
+            $progress->start();
+
+            foreach ($deleting as $docId) {
+                $this->solrTools->removeDocument($docId);
+                $progress->advance();
+            }
+
+            $progress->finish();
+        } else
+            $output->writeln(' <info>found no orphan</info>');
     }
 }
 

diff --git a/lib/Controller/SettingsController.php b/lib/Controller/SettingsController.php
@@ -64,7 +64,7 @@ public function __construct($appName, IRequest $request, ConfigService $configSe
      */
     public function index()
     {
-        $documentsCount = $this->solrAdmin->count($error);
+        $documentsCount = $this->solrTools->count($error);
 
         $params = [
             'configured' => $this->configService->getAppValue('configured'),