Merge pull request #30 from daita/0.4.x0.4.2

0.4.2:
- extract and index files from trash
- index remove orphans

12 files changed, 287 insertions, 89 deletions

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 6241b21..2f0f075 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,16 @@
+**v0.4.2**
+- extract and index files from trash
+- index remove orphans
+
+
+**v0.4.1**
+- Force index unlock after 24h
+- maximum size on extracted file
+- new field nextant_path
+- documents with its file deleted are removed
+- bugfixes
+
+
 **v0.4.0**
 - index safe guard
 - new options on nextant:index
diff --git a/appinfo/info.xml b/appinfo/info.xml
index 8ec1d83..7652d7a 100644
--- a/appinfo/info.xml
+++ b/appinfo/info.xml
@@ -13,7 +13,7 @@ _Recognized file format: text, rtf, pdf, html, openoffice, office, ..._
 
 
 	</description>
-	<version>0.4.1</version>
+	<version>0.4.2</version>
 	<licence>agpl</licence>
 	<author>Maxence Lange</author>
 	<types>
diff --git a/appinfo/register_command.php b/appinfo/register_command.php
index 6e99dc2..6429809 100644
--- a/appinfo/register_command.php
+++ b/appinfo/register_command.php
@@ -27,7 +27,7 @@
 $app = new \OCA\Nextant\AppInfo\Application();
 $c = $app->getContainer();
 
-$application->add(new OCA\Nextant\Command\Check($c->query('SolrService'), $c->query('SolrAdminService')));
+$application->add(new OCA\Nextant\Command\Check($c->query('SolrService'), $c->query('SolrToolsService'), $c->query('SolrAdminService')));
 $application->add(new OCA\Nextant\Command\Clear($c->query('ConfigService'), $c->query('SolrService'), $c->query('SolrAdminService')));
 $application->add(new OCA\Nextant\Command\Scan(OC::$server->getUserManager(), $c->query('UserFolder'), $c->query('SolrService'), $c->query('FileService')));
 $application->add(new OCA\Nextant\Command\Index(OC::$server->getUserManager(), $c->query('RootFolder'), $c->query('SolrService'), $c->query('SolrToolsService'), $c->query('ConfigService'), $c->query('FileService'), $c->query('MiscService')));
diff --git a/lib/Command/Check.php b/lib/Command/Check.php
index aa0c790..d388f7c 100644
--- a/lib/Command/Check.php
+++ b/lib/Command/Check.php
@@ -37,12 +37,15 @@ class Check extends Base
 
     private $solrService;
 
+    private $solrTools;
+
     private $solrAdmin;
 
-    public function __construct($solrService, $solrAdmin)
+    public function __construct($solrService, $solrTools, $solrAdmin)
     {
         parent::__construct();
         $this->solrService = $solrService;
+        $this->solrTools = $solrTools;
         $this->solrAdmin = $solrAdmin;
     }
 
@@ -53,11 +56,10 @@ class Check extends Base
     }
 
     protected function execute(InputInterface $input, OutputInterface $output)
-    {       
-        if (!$this->solrService->configured())
-        {
+    {
+        if (! $this->solrService->configured()) {
             $output->writeln('Nextant is not yet configured');
-            return;            
+            return;
         }
         
         $this->solrService->setOutput($output);
@@ -75,7 +77,7 @@ class Check extends Base
             return false;
         }
         
-        $output->writeln('Your solr contains ' . $this->solrAdmin->count() . ' documents');
+        $output->writeln('Your solr contains ' . $this->solrTools->count() . ' documents');
     }
 }
 
diff --git a/lib/Command/Index.php b/lib/Command/Index.php
index ceb6db9..67b8bb8 100644
--- a/lib/Command/Index.php
+++ b/lib/Command/Index.php
@@ -26,7 +26,6 @@
  */
 namespace OCA\Nextant\Command;
 
-use \OCA\Nextant\Service\SolrToolsService;
 use \OCA\Nextant\Service\FileService;
 use OC\Core\Command\Base;
 use Symfony\Component\Console\Input\InputArgument;
@@ -108,10 +107,12 @@ class Index extends Base
         
         $this->configService->setAppValue('solr_lock', time());
         
+        $documentIds = array();
+        
         //
         // extract files
         $output->writeln('');
-        $output->writeln('* Extracting new files to Solr:');
+        $output->writeln('* Extracting files to Solr:');
         $output->writeln('');
         
         $users = $this->userManager->search('');
@@ -130,17 +131,22 @@ class Index extends Base
             $this->solrService->setOwner($userId);
             
             $this->miscService->debug('Init Extracting new files for user ' . $userId);
-            $result = $this->browseUserDirectory($userId, $output);
+            $result_files = $this->browseUserDirectory($output, $userId, '/files', array(), $docIds_files);
             
-            if ($result['total'] > 0) {
-                
+            $output->writeln('');
+            $result_trash = $this->browseUserDirectory($output, $userId, '/files_trashbin', array(
+                'deleted' => true
+            ), $docIds_trash);
+            
+            if (($result_files['total'] + $result_trash['total']) > 0) {
+                $documentIds = array_merge($documentIds, $docIds_files, $docIds_trash);
                 array_push($extractedDocuments, array(
                     'userid' => $userId,
-                    'files' => $result['files']
+                    'files' => array_merge($result_files['files'], $result_trash['files'])
                 ));
                 
-                $extractedTotal += sizeof($result['files']);
-                $processedTotal += $result['processed'];
+                $extractedTotal += sizeof($result_files['files']) + sizeof($result_trash['files']);
+                $processedTotal += $result_files['processed'] + $result_trash['processed'];
             }
             
             $output->writeln('');
@@ -152,9 +158,12 @@ class Index extends Base
         // update Documents
         $output->writeln('');
         $output->writeln('* Updating documents status');
+        $output->writeln('');
+        
         $noFailure = true;
         $usersTotal = sizeof($extractedDocuments);
         $usersCurrent = 0;
+        $processedFile = 0;
         foreach ($extractedDocuments as $doc) {
             $usersCurrent ++;
             
@@ -164,13 +173,27 @@ class Index extends Base
             
             $this->miscService->debug('Init Updating documents for user ' . $userId);
             
-            if (! $this->updateUserDocuments($userId, $fileIds, $output))
+            if (! $this->updateUserDocuments($userId, $fileIds, $output, $updateDocs))
                 $noFailure = false;
             
+            $processedFile += $updateDocs;
             $output->writeln('');
         }
         
+        $output->writeln('       ' . $processedFile . ' file(s) updated');
+        
+        // $output->writeln(' - ' . $deleted . ' documents removed');
+        
         Filesystem::tearDown();
+        
+        //
+        // removing orphan
+        $output->writeln('');
+        $output->writeln('* Removing orphan documents');
+        $output->writeln('');
+        
+        $this->removeOrphans($output, $documentIds);
+        
         $this->configService->needIndex(false);
         
         if ($noFailure)
@@ -179,20 +202,22 @@ class Index extends Base
             $this->configService->needIndex(true);
         
         $this->configService->setAppValue('solr_lock', '0');
+        
+        $output->writeln('');
     }
 
-    private function browseUserDirectory($userId, $output)
+    private function browseUserDirectory($output, $userId, $dir, $options, &$docIds)
     {
         Filesystem::tearDown();
         Filesystem::init($userId, '');
         $this->fileService->setView(Filesystem::getView());
         $this->miscService->debug('(' . $userId . ') - Init Filesystem');
         
-        $userFolder = FileService::getUserFolder($this->rootFolder, $userId, '/files');
+        $userFolder = FileService::getUserFolder($this->rootFolder, $userId, $dir);
         if ($userFolder != null && $userFolder) {
             $folder = $userFolder->get('/');
             
-            $this->miscService->debug('(' . $userId . ') - found root folder');
+            $this->miscService->debug('(' . $userId . '/' . $dir . ') - found root folder');
             $files = $folder->search('');
         } else
             $files = array();
@@ -200,10 +225,10 @@ class Index extends Base
         $this->miscService->debug('(' . $userId . ') - found ' . sizeof($files) . ' files');
         
         $progress = new ProgressBar($output, sizeof($files));
-        $progress->setMessage('<info>' . $userId . '</info>: ');
+        $progress->setMessage('<info>' . $userId . '</info>' . $dir . ': ');
         $progress->setMessage('', 'jvm');
         $progress->setMessage('[preparing]', 'infos');
-        $progress->setFormat(" %message:-30s%%current:5s%/%max:5s% [%bar%] %percent:3s%% \n    %infos:1s% %jvm:-30s%      ");
+        $progress->setFormat(" %message:-38s%%current:5s%/%max:5s% [%bar%] %percent:3s%% \n    %infos:1s% %jvm:-30s%      ");
         $progress->start();
         
         if (sizeof($files) > 10)
@@ -211,10 +236,11 @@ class Index extends Base
         
         $filesProcessed = 0;
         $fileIds = array();
+        $docIds = array();
         $lastProgressTick = 0;
         foreach ($files as $file) {
             
-            $this->miscService->debug('(' . $userId . ') - scaning file #' . $file->getId() . ' (' . $file->getMimeType() . ') ' . $file->getPath());
+            $this->miscService->debug('(' . $userId . ') - scanning file #' . $file->getId() . ' (' . $file->getMimeType() . ') ' . $file->getPath());
             
             if ($this->hasBeenInterrupted()) {
                 $this->configService->setAppValue('solr_lock', '0');
@@ -233,10 +259,11 @@ class Index extends Base
                 
                 $forceExtract = false;
                 $status = 0;
-                
                 if ($this->fileService->addFileFromPath($file->getPath(), $forceExtract, $status)) {
+                    array_push($docIds, (int) $file->getId());
                     array_push($fileIds, array(
                         'fileid' => $file->getId(),
+                        'options' => $options,
                         'path' => $file->getPath()
                     ));
                     $filesProcessed += $status;
@@ -261,7 +288,7 @@ class Index extends Base
         );
     }
 
-    private function updateUserDocuments($userId, $fileIds, $output)
+    private function updateUserDocuments($userId, $fileIds, $output, &$processedfile)
     {
         Filesystem::tearDown();
         Filesystem::init($userId, '');
@@ -271,7 +298,7 @@ class Index extends Base
         // $cycle = array_chunk($fileIds, 5);
         
         $progress = new ProgressBar($output, sizeof($fileIds));
-        $progress->setFormat(" %message:-30s%%current:5s%/%max:5s%  [%bar%] %percent:3s%% \n    %infos:1s% %jvm:-30s% %failures:1s%     ");
+        $progress->setFormat(" %message:-38s%%current:5s%/%max:5s%  [%bar%] %percent:3s%% \n    %infos:1s% %jvm:-30s% %failures:1s%     ");
         $progress->setMessage('<info>' . $userId . '</info>: ');
         $progress->setMessage('', 'jvm');
         $progress->setMessage('', 'failures');
@@ -283,27 +310,32 @@ class Index extends Base
         sleep(5);
         $i = 0;
         $lastProgressTick = 0;
-        $failure = 0;
-        foreach ($fileIds as $file) {
+        $failureIds = array();
+        $processedfile = 0;
+        while ($file = array_shift($fileIds)) {
+            
             if ($this->hasBeenInterrupted()) {
                 $this->configService->setAppValue('solr_lock', '0');
                 throw new \Exception('ctrl-c');
             }
             
-            $result = $this->fileService->updateFiles(array(
+            $count = $this->fileService->updateFiles(array(
                 $file
-            ));
-            $progress->setMessage('failure(s): ' . $failure, 'failures');
+            ), $file['options']);
             
-            if ($result)
-                $this->miscService->debug('' . $userId . ' update done');
-            else {
-                $failure ++;
-                $progress->setMessage('failure(s): ' . $failure, 'failures');
+            // $progress->setMessage('failure(s): ' . $failure, 'failures');
+            
+            if ($count === false) {
+                array_push($failureIds, $file);
+                $progress->setMessage('failure(s): ' . sizeof($failureIds), 'failures');
                 $this->miscService->debug('' . $userId . ' update failed');
+            } else {
+                $processedfile += $count;
+                $this->miscService->debug('' . $userId . ' update done');
             }
             
             $progress->setMessage('[updating] -', 'infos');
+            
             if ((time() - self::REFRESH_INFO_SYSTEM) > $lastProgressTick) {
                 $infoSystem = $this->solrTools->getInfoSystem();
                 $progress->setMessage($infoSystem->jvm->memory->used, 'jvm');
@@ -317,7 +349,7 @@ class Index extends Base
             
             $i ++;
             
-            // let's take a break every 1000 files
+            // let's take a break every 500 files
             if (($i % 500) == 0) {
                 $progress->setMessage('[standby] -', 'infos');
                 $progress->display();
@@ -325,9 +357,64 @@ class Index extends Base
             }
         }
         
+        $progress->setMessage('', 'jvm');
+        $progress->setMessage('', 'infos');
+        $progress->finish();
+        
+        return (sizeof($failureIds) == 0);
+    }
+
+    private function removeOrphans($output, $fileIds)
+    {
+        $progress = new ProgressBar($output, $this->solrTools->count());
+        
+        $progress->setMessage('<info>spoting orphans</info>:');
+        $progress->setFormat(" %message:-51s%[%bar%] %percent:3s%%");
+        $progress->start();
+        
+        $deleting = array();
+        $page = 0;
+        while (true) {
+            
+            if ($this->hasBeenInterrupted()) {
+                $this->configService->setAppValue('solr_lock', '0');
+                throw new \Exception('ctrl-c');
+            }
+            
+            $ids = $this->solrTools->getAll($page, $lastPage, $error);
+            if (! $ids)
+                return false;
+            
+            foreach ($ids as $id) {
+                
+                if (! in_array($id, $fileIds) && (! in_array($id, $deleting)))
+                    array_push($deleting, $id);
+                
+                $progress->advance();
+            }
+            
+            if ($lastPage)
+                break;
+            $page ++;
+        }
+        
         $progress->finish();
+        $output->writeln('');
         
-        return ($failure == 0);
+        if (sizeof($deleting) > 0) {
+            $progress = new ProgressBar($output, sizeof($deleting));
+            $progress->setMessage('<info>removing orphans</info>:');
+            $progress->setFormat(" %message:-38s%%current:5s%/%max:5s%  [%bar%] %percent:3s%%");
+            $progress->start();
+            
+            foreach ($deleting as $docId) {
+                $this->solrTools->removeDocument($docId);
+                $progress->advance();
+            }
+            
+            $progress->finish();
+        } else
+            $output->writeln(' <info>found no orphan</info>');
     }
 }
 
diff --git a/lib/Controller/SettingsController.php b/lib/Controller/SettingsController.php
index cd2c968..578a8e9 100644
--- a/lib/Controller/SettingsController.php
+++ b/lib/Controller/SettingsController.php
@@ -64,7 +64,7 @@ class SettingsController extends Controller
      */
     public function index()
     {
-        $documentsCount = $this->solrAdmin->count($error);
+        $documentsCount = $this->solrTools->count($error);
         
         $params = [
             'configured' => $this->configService->getAppValue('configured'),
diff --git a/lib/Cron/BackgroundIndex.php b/lib/Cron/BackgroundIndex.php
index 682a90b..ec20aeb 100644
--- a/lib/Cron/BackgroundIndex.php
+++ b/lib/Cron/BackgroundIndex.php
@@ -51,11 +51,11 @@ class BackgroundIndex extends \OC\BackgroundJob\TimedJob
         $this->miscService = $c->query('MiscService');
         $this->userManager = $c->query('UserManager');
         $this->solrService = $c->query('SolrService');
+        $this->solrTools = $c->query('SolrToolsService');
         $this->fileService = $c->query('FileService');
         $this->rootFolder = $c->query('RootFolder');
-        
-        // $this->setDebug(true);
-        
+            
+            // $this->setDebug(true);
         if (! $this->configService->neededIndex()) {
             $this->miscService->debug('Looks like there is no need to index');
             return;
@@ -90,30 +90,65 @@ class BackgroundIndex extends \OC\BackgroundJob\TimedJob
         $users = $this->userManager->search('');
         $extractedDocuments = array();
         
+        $documentIds = array();
         $noFailure = true;
         foreach ($users as $user) {
             
             $userId = $user->getUID();
             $this->solrService->setOwner($userId);
             
-            $result = $this->browseUserDirectory($userId);
-            if (! $result) {
+            $result_files = $this->browseUserDirectory($userId, '/files', array(), $docIds_files);
+            $result_trash = $this->browseUserDirectory($userId, '/files_trashbin', array(
+                'deleted' => true
+            ), $docIds_trash);
+            
+            if (! $result_files || ! $result_trash) {
                 $this->miscService->log('Background index had some issue', 2);
                 $noFailure = false;
             }
+            $documentIds = array_merge($documentIds, $docIds_files, $docIds_trash);
+        }
+        
+        // orphans
+        $deleting = array();
+        $page = 0;
+        while (true) {
+            
+            $ids = $this->solrTools->getAll($page, $lastPage, $error);
+            if (! $ids)
+                break;
+            
+            foreach ($ids as $id) {
+                if (! in_array($id, $documentIds) && (! in_array($id, $deleting)))
+                    array_push($deleting, $id);
+            }
+            
+            if ($lastPage)
+                break;
+            
+            $page ++;
+            if ($page > 10000) {
+                $this->miscService->log('Looks like we reached a 1,000,000 documents');
+                break;
+            }
         }
         
+        foreach ($deleting as $docId)
+            $this->solrTools->removeDocument($docId);
+        
         return $noFailure;
     }
 
-    private function browseUserDirectory($userId)
+    private function browseUserDirectory($userId, $dir, $options, &$docIds)
     {
+        $docIds = array();
+        
         Filesystem::tearDown();
         Filesystem::init($userId, '');
         
         $this->fileService->setView(Filesystem::getView());
         
-        $userFolder = FileService::getUserFolder($this->rootFolder, $userId, '/files');
+        $userFolder = FileService::getUserFolder($this->rootFolder, $userId, $dir);
         if ($userFolder == null || ! $userFolder)
             return true;
         
@@ -131,8 +166,10 @@ class BackgroundIndex extends \OC\BackgroundJob\TimedJob
             if (! $file->isShared() && $file->getType() == \OCP\Files\FileInfo::TYPE_FILE) {
                 if (($this->fileService->addFileFromPath($file->getPath(), false))) {
                     
+                    array_push($docIds, (int) $file->getId());
                     array_push($fileIds, array(
                         'fileid' => $file->getId(),
+                        'options' => $options,
                         'path' => $file->getPath()
                     ));
                 }
@@ -141,15 +178,16 @@ class BackgroundIndex extends \OC\BackgroundJob\TimedJob
         
         sleep(5);
         $i = 0;
+        $currentIndex = time();
         foreach ($fileIds as $file) {
             $i ++;
             
             $this->miscService->debug('Cron update ' . $i . ' - file #' . $file['fileid']);
             $result = $this->fileService->updateFiles(array(
                 $file
-            ));
+            ), $file['options']);
             
-            if (! $result) {
+            if ($result === false) {
                 $this->miscService->log('Failed to update files flag during background jobs (file #' . $file['fileid'] . ')', 3);
                 $noFailure = false;
                 sleep(10);
diff --git a/lib/Events/FilesEvents.php b/lib/Events/FilesEvents.php
index 14176a3..9b89f2d 100644
--- a/lib/Events/FilesEvents.php
+++ b/lib/Events/FilesEvents.php
@@ -109,7 +109,7 @@ class FilesEvents
             else
                 $this->configService->needIndex(true);
         } else {
-            if ($this->configService->getAppValue('live_extract') == '1')
+            if ($this->configService->getAppValue('live_docupdate') == '1')
                 $this->fileService->removeFiles($path);
             else
                 $this->configService->needIndex(true);
diff --git a/lib/Service/FileService.php b/lib/Service/FileService.php
index 254c68d..4e03d05 100644
--- a/lib/Service/FileService.php
+++ b/lib/Service/FileService.php
@@ -91,7 +91,7 @@ class FileService
             return false;
         }
         
-        if (! SolrService::extractableFile($fileInfo->getMimeType()))
+        if (! SolrService::extractableFile($fileInfo->getMimeType(), $path))
             return false;
         
         if (! $forceExtract && $this->solrTools->isDocumentUpToDate($fileInfo->getId(), $fileInfo->getMTime())) {
@@ -141,7 +141,7 @@ class FileService
                 $subfiles = $this->view->getDirectoryContent($file['path']);
                 foreach ($subfiles as $subfile) {
                     $result = $this->updateFiles($subfile->getId(), $options, false);
-                    if ($result == false)
+                    if ($result === false)
                         return false;
                     $pack = array_merge($pack, $result);
                     
@@ -162,12 +162,12 @@ class FileService
         if (! $isRoot)
             return $pack;
         
-        $solrResult = $this->solrTools->updateDocuments($pack, $error);
+        $count = $this->solrTools->updateDocuments($pack, $error);
         
-        if (! $solrResult)
+        if ($count === false)
             $this->configService->needIndex(true);
         
-        return $solrResult;
+        return $count;
     }
 
     /**
diff --git a/lib/Service/SolrAdminService.php b/lib/Service/SolrAdminService.php
index bb1e71c..03a023e 100644
--- a/lib/Service/SolrAdminService.php
+++ b/lib/Service/SolrAdminService.php
@@ -380,37 +380,5 @@ class SolrAdminService
         
         return false;
     }
-
-    /**
-     * Count document on Solr Core
-     *
-     * @param number $error            
-     * @return boolean
-     */
-    public function count(&$error = 0)
-    {
-        if (! $this->solrService || ! $this->solrService->configured() || ! $this->solrService->getClient())
-            return false;
-        
-        $client = $this->solrService->getClient();
-        
-        try {
-            $query = $client->createSelect();
-            $query->setQuery('*:*');
-            $query->setRows(0);
-            $resultset = $client->execute($query);
-            
-            return $resultset->getNumFound();
-        } catch (\Solarium\Exception\HttpException $ehe) {
-            if ($ehe->getStatusMessage() == 'OK')
-                $error = SolrService::EXCEPTION_SOLRURI;
-            else
-                $error = SolrService::EXCEPTION_HTTPEXCEPTION;
-        } catch (\Solarium\Exception $e) {
-            $error = SolrService::EXCEPTION;
-        }
-        
-        return false;
-    }
 }
     
\ No newline at end of file
diff --git a/lib/Service/SolrService.php b/lib/Service/SolrService.php
index a14faf4..afbc76d 100644
--- a/lib/Service/SolrService.php
+++ b/lib/Service/SolrService.php
@@ -149,7 +149,7 @@ class SolrService
         $this->groups = $groups;
     }
 
-    public static function extractableFile($mimetype)
+    public static function extractableFile($mimetype, $path = '')
     {
         switch (FileService::getBaseTypeFromMime($mimetype)) {
             case 'text':
@@ -165,6 +165,12 @@ class SolrService
             
             case 'application/rtf':
                 return true;
+            
+            case 'application/octet-stream':
+                $ext = pathinfo($path, PATHINFO_EXTENSION);
+                if (substr($ext, 0, 1) == 'd' && ((int) (substr($ext, 1)) > 0))
+                    return true;
+                return false;
         }
         
         $acceptedMimeType = array(
diff --git a/lib/Service/SolrToolsService.php b/lib/Service/SolrToolsService.php
index faadef7..7e43b61 100644
--- a/lib/Service/SolrToolsService.php
+++ b/lib/Service/SolrToolsService.php
@@ -38,6 +38,8 @@ use \OCA\Nextant\Service\MiscService;
 class SolrToolsService
 {
 
+    const GETALL_ROWS = 100;
+
     const UPDATE_MAXIMUM_QUERYTIME = 2000;
 
     const UPDATE_MAXIMUM_FILEPROCESS = 15;
@@ -204,13 +206,14 @@ class SolrToolsService
                     return false;
                 }
                 
+                $documentProcessed += sizeof($docs);
+                
                 if ($request->getQueryTime() > self::UPDATE_MAXIMUM_QUERYTIME) {
                     $this->miscService->log('Maximum Update Query Time (' . self::UPDATE_MAXIMUM_QUERYTIME . 'ms) reached, standby.', 1);
                     return false;
                     // sleep(10);
                 }
                 
-                $documentProcessed += sizeof($docs);
                 if ($documentProcessed >= self::UPDATE_MAXIMUM_FILEPROCESS) {
                     $this->miscService->log('Maximum number of processed files (' . self::UPDATE_MAXIMUM_FILEPROCESS . ') reached, we won\'t go any further.', 2);
                     $error = SolrService::EXCEPTION_UPDATE_MAXIMUM_REACHED;
@@ -218,7 +221,7 @@ class SolrToolsService
                 }
             }
             
-            return true;
+            return $documentProcessed;
         } catch (\Solarium\Exception\HttpException $ehe) {
             if ($ehe->getStatusMessage() == 'OK')
                 $error = SolrService::EXCEPTION_UPDATE_FIELD_FAILED;
@@ -331,6 +334,55 @@ class SolrToolsService
     }
 
     /**
+     * return ids of all documents
+     *
+     * @param number $page            
+     * @param boolean $lastpage            
+     * @param number $error            
+     * @return boolean
+     */
+    public function getAll($page, &$lastpage = false, &$error = 0)
+    {
+        if (! $this->solrService || ! $this->solrService->configured() || ! $this->solrService->getClient())
+            return false;
+        
+        $client = $this->solrService->getClient();
+        
+        try {
+            $query = $client->createSelect();
+            $query->setQuery('id:*');
+            $query->addSort('id', $query::SORT_ASC);
+            $query->setStart($page);
+            $query->setRows(self::GETALL_ROWS);
+            $query->setFields(array(
+                'id'
+            ));
+            
+            $resultset = $client->execute($query);
+            
+            $ids = array();
+            foreach ($resultset as $document) {
+                $docid = (int) $document->id;
+                array_push($ids, $docid);
+            }
+            
+            if ((($page + 1) * self::GETALL_ROWS) >= $resultset->getNumFound())
+                $lastpage = true;
+            
+            return $ids;
+        } catch (\Solarium\Exception\HttpException $ehe) {
+            if ($ehe->getStatusMessage() == 'OK')
+                $error = SolrService::EXCEPTION_SOLRURI;
+            else
+                $error = SolrService::EXCEPTION_HTTPEXCEPTION;
+        } catch (\Solarium\Exception $e) {
+            $error = SolrService::EXCEPTION;
+        }
+        
+        return false;
+    }
+
+    /**
      * Check the mtime of a file and return if document is up to date
      *
      * @param number $docid            
@@ -402,5 +454,37 @@ class SolrToolsService
         
         return $result;
     }
+
+    /**
+     * Count document on Solr Core
+     *
+     * @param number $error            
+     * @return boolean
+     */
+    public function count(&$error = 0)
+    {
+        if (! $this->solrService || ! $this->solrService->configured() || ! $this->solrService->getClient())
+            return false;
+        
+        $client = $this->solrService->getClient();
+        
+        try {
+            $query = $client->createSelect();
+            $query->setQuery('id:*');
+            $query->setRows(0);
+            $resultset = $client->execute($query);
+            
+            return $resultset->getNumFound();
+        } catch (\Solarium\Exception\HttpException $ehe) {
+            if ($ehe->getStatusMessage() == 'OK')
+                $error = SolrService::EXCEPTION_SOLRURI;
+            else
+                $error = SolrService::EXCEPTION_HTTPEXCEPTION;
+        } catch (\Solarium\Exception $e) {
+            $error = SolrService::EXCEPTION;
+        }
+        
+        return false;
+    }
 }
     
\ No newline at end of file