Improve import-all functionality

This switches the import-all function to work with any number of images and without timing out by paging through a user's photos and importing each photo. The API endpoints are renamed (as a new one is introduced). Tag importing is fixed to use the 'raw' value rather than the spaceless normalized one. Checksums of files are also checked before import, along with filenames. Logging is added throughout the import process. Refs: #12
Piwigo · Mar 22, 2018 · 2a486c9 · 2a486c9
1 parent 16b1840
commit 2a486c9
Show file tree

Hide file tree

Showing 16 changed files with 501 additions and 278 deletions.
diff --git a/admin/import.php b/admin/import.php
@@ -244,68 +244,21 @@
   // list all photos of the user
   case 'list_all':
   {
-    $flickr_prefix = 'flickr-'.$u['username'].'-';
-
-    // get all photos in all albums
-    $all_albums = $flickr->photosets_getList($u['id']);
-    $all_albums = $all_albums['photoset'];
-
-    $all_photos = array();
-    foreach ($all_albums as $album)
-    {
-      $album_photos = $flickr->photosets_getPhotos($album['id'], NULL, NULL, 500, NULL, 'photos');
-      $album_photos = $album_photos['photoset']['photo'];
-
-      foreach ($album_photos as $photo)
-      {
-        $all_photos[ $photo['id'] ][] = $album['title'];
-      }
-    }
-
-    // get existing photos
-    $query = '
-SELECT id, file
-  FROM '.IMAGES_TABLE.'
-  WHERE file LIKE "'.$flickr_prefix.'%"
-;';
-    $existing_photos = simple_hash_from_query($query, 'id', 'file');
-    $existing_photos = array_map(create_function('$p', 'return preg_replace("#^'.$flickr_prefix.'([0-9]+)\.([a-z]{3,4})$#i", "$1", $p);'), $existing_photos);
-
-    // remove existing photos
-    $duplicates = 0;
-    foreach ($all_photos as $id => &$photo)
-    {
-      if (in_array($id, $existing_photos))
-      {
-        unset($all_photos[$id]);
-        $duplicates++;
-      }
-      else
-      {
-        $photo = array(
-          'id' => $id,
-          'albums' => implode(',', $photo),
-          );
-      }
-    }
-    unset($photo);
-    $all_photos = array_values($all_photos);
-
-    if ($duplicates>0)
-    {
-      $page['infos'][] = '<a href="admin.php?page=batch_manager&amp;filter=prefilter-flickr">'
-          .l10n_dec(
-            'One picture is not displayed because already existing in the database.',
-            '%d pictures are not displayed because already existing in the database.',
-            $duplicates)
-        .'</a>';
-    }
-
-    $template->assign(array(
-      'nb_elements' => count($all_photos),
-      'all_elements' => json_encode($all_photos),
+    $template->func_combine_script([
+      'id' => 'plugins.flickr2piwigo.list_all',
+      'path' => FLICKR_PATH.'/admin/template/list_all.js',
+      'require' => 'jquery.ajaxmanager,jquery.jgrowl',
+      'load' => 'footer',
+    ]);
+
+    $photoInfo = $flickr->people()->getPhotos($u['id']);
+
+    $template->assign([
+      'flickrUserId' => $u['id'],
+      'nb_elements' => (int)$photoInfo['total'],
       'F_ACTION' => FLICKR_ADMIN . '-import&amp;action=import_set',
-      ));
+      'CACHE_KEYS' => get_admin_client_cache_keys(['categories']),
+    ]);
 
     // get piwigo categories
     $query = '

diff --git a/admin/template/import.list_all.tpl b/admin/template/import.list_all.tpl
@@ -1,125 +1,13 @@
-{footer_script require='jquery.ajaxmanager,jquery.jgrowl'}
-{* <!-- CATEGORIES --> *}
-var categoriesCache = new CategoriesCache({
-  serverKey: '{$CACHE_KEYS.categories}',
-  serverId: '{$CACHE_KEYS._hash}',
-  rootUrl: '{$ROOT_URL}'
-});
+{footer_script}
 
-categoriesCache.selectize(jQuery('[data-selectize=categories]'), {
-  filter: function(categories, options) {
-    if (categories.length > 0) {
-      jQuery("#albumSelection").show();
-    }
-
-    return categories;
-  }
-});
+var cacheKeysCategories = '{$CACHE_KEYS.categories}';
+var cacheKeysHash = '{$CACHE_KEYS._hash}';
+var rootUrl = '{$ROOT_URL}';
+var photosTotal = {$nb_elements};
+var flickrUserId = '{$flickrUserId}';
 
-jQuery('[data-add-album]').pwgAddAlbum({
-  cache: categoriesCache,
-  afterSelect: function() {
-    jQuery("#albumSelection").show();
-  }
-});
-
-(function($){
-  /* global vars */
-  var all_elements = {$all_elements};
-  var import_done = 0;
-  var import_selected = {$nb_elements};
-  var queuedManager = $.manageAjax.create('queued', {
-    queue: true,
-    maxRequests: 1
-  });
-
-  /* import queue */
-  function performImport(photo, album, fills) {
-    queuedManager.add({
-      type: 'GET',
-      dataType: 'json',
-      url: 'ws.php',
-      data: {
-        method: 'pwg.images.addFlickr',
-        id: photo,
-        category: album,
-        fills: fills,
-        format: 'json'
-      },
-      success: function(data) {
-        if (data['stat'] == 'ok') {
-          $.jGrowl(data['result'], {
-            theme: 'success', life: 4000, sticky: false,
-            header: '{'Success'|translate}',
-          });
-          $("#photo-"+photo.id).fadeOut(function(){ $(this).remove(); });
-        }
-        else {
-          $.jGrowl(data['result'], {
-            theme: 'error', sticky: true,
-            header: '{'ERROR'|translate}'
-          });
-        }
-
-        import_done++;
-        $("#progress").html(import_done +"/"+ import_selected);
-
-        if (import_done == import_selected) {
-          $("#import_form").append('<input type="hidden" name="done" value="' + import_done + '">');
-          $("#import_form").submit();
-        }
-      },
-      error: function(data) {
-        $.jGrowl('{'an error happened'|translate|escape:'javascript'}', {
-          theme: 'error', sticky: true,
-          header: '{'ERROR'|translate}'
-        });
-      }
-    });
-  }
-
-  /* begin import */
-  $('#beginImport').click(function() {
-    $("#loader_import").fadeIn();
-
-    if ($("input[name='album_mode']:checked").val() == 'identical') {
-      var album = 0;
-    }
-    else {
-      var album = $('select[name=category] option:selected').val();
-    }
-
-    var fills = '';
-    $("input[name^='fill_']:checked").each(function() {
-      fills+= $(this).attr("name") +',';
-    });
-
-    import_selected = all_elements.length;
-    $("#progress").html("0/"+ import_selected);
-
-    for (var i in all_elements) {
-      if (album == 0) this_album = all_elements[i]['albums'];
-      else            this_album = album;
-
-      performImport(all_elements[i]['id'], this_album, fills);
-    }
-
-    return false;
-  });
-
-  /* album mode */
-  $("input[name='album_mode']").change(function() {
-    if ($(this).val() == 'one_album') {
-      $("#albumSelectWrapper").slideDown();
-    }
-    else {
-      $("#albumSelectWrapper").slideUp();
-    }
-  });
-}(jQuery));
 {/footer_script}
 
-
 <form action="{$F_ACTION}" method="post" id="import_form">
 
   <fieldset>
@@ -137,7 +25,7 @@ jQuery('[data-add-album]').pwgAddAlbum({
 
     <p>
       <label><input type="radio" name="album_mode" value="identical" checked="checked"> {'Reproduce flickr albums'|translate}</label><br>
-      <label><input type="radio" name="album_mode" value="one_album"> {'Import all photos in this album'|translate} :</label>
+      <label><input type="radio" name="album_mode" value="one_album"> {'Import all photos in this album'|translate}:</label>
     </p>
 
     <p id="albumSelectWrapper" style="display:none;">
@@ -160,8 +48,11 @@ jQuery('[data-add-album]').pwgAddAlbum({
     </p>
 
     <p>
-      <input type="submit" name="import_set" id="beginImport" value="{'Begin transfer'|translate}" {if not $nb_elements}style="display:none;"{/if}>
-      <span id="loader_import" style="display:none;"><img src="admin/themes/default/images/ajax-loader.gif"> <i>{'Processing...'|translate}</i> <span id="progress"></span></span>
+      <button type="button" name="import_set" id="beginImport" {if not $nb_elements}style="display:none;"{/if}>{'Begin transfer'|translate}</button>
+    </p>
+    <p id="loader_import" style="display:none;">
+      <img src="admin/themes/default/images/ajax-loader.gif"> <i>{'Processing...'|translate}</i> <span id="progress"></span>
+      <button type="button" class="stop">{'Stop'|translate}</button>
     </p>
   </fieldset>
-</form>
+</form>
diff --git a/admin/template/import.list_photos.tpl b/admin/template/import.list_photos.tpl
@@ -135,7 +135,7 @@ jQuery('[data-add-album]').pwgAddAlbum({
       dataType: 'json',
       url: 'ws.php',
       data: {
-        method: 'pwg.images.addFlickr',
+        method: 'flickr2piwigo.importPhoto',
         id: photo,
         category: album,
         fills: fills,

diff --git a/admin/template/import.tpl b/admin/template/import.tpl
@@ -38,9 +38,8 @@ jQuery('.load').click(function() {
 <p>
   <input type="submit" data="{$list_albums_url}" class="load" value="{'List my albums'|translate}">
   <input type="submit" data="{$import_all_url}" class="load" value="{'Import all my pictures'|translate}">
-  <br>
-  <span id="loader_import" style="display:none;"><img src="admin/themes/default/images/ajax-loader.gif"> <i>{'Processing...'|translate}</i></span>
 </p>
+<p id="loader_import" style="display:none;"><img src="admin/themes/default/images/ajax-loader.gif"> <i>{'Processing...'|translate}</i></p>
 
 {* <!-- ALBUMS LIST --> *}
 {else if $ACTION == 'list_albums'}
@@ -60,7 +59,7 @@ jQuery('.load').click(function() {
   </li>
 {/foreach}
 </ul>
-<span id="loader_import" style="display:none;"><img src="admin/themes/default/images/ajax-loader.gif"> <i>{'Processing...'|translate}</i></span>
+<p id="loader_import" style="display:none;"><img src="admin/themes/default/images/ajax-loader.gif"> <i>{'Processing...'|translate}</i></p>
 
 {* <!-- PHOTOS LIST --> *}
 {else if $ACTION == 'list_photos'}