Skip to content

Commit

Permalink
Improve import-all functionality
Browse files Browse the repository at this point in the history 
This switches the import-all function to work with any number of
images and without timing out by paging through a user's photos
and importing each photo. The API endpoints are renamed (as a
new one is introduced).

Tag importing is fixed to use the 'raw' value rather than the
spaceless normalized one.

Checksums of files are also checked before import, along with
filenames.

Logging is added throughout the import process.

A button is added to stop the import before it's finished.

Refs: #12
  • Loading branch information
@samwilson
samwilson committed Mar 25, 2018
1 parent 16b1840 commit dd90fac
Show file tree
Hide file tree
Showing 16 changed files with 509 additions and 278 deletions.
75 changes: 14 additions & 61 deletions admin/import.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -244,68 +244,21 @@
// list all photos of the user
case 'list_all':
{
$flickr_prefix = 'flickr-'.$u['username'].'-';

// get all photos in all albums
$all_albums = $flickr->photosets_getList($u['id']);
$all_albums = $all_albums['photoset'];

$all_photos = array();
foreach ($all_albums as $album)
{
$album_photos = $flickr->photosets_getPhotos($album['id'], NULL, NULL, 500, NULL, 'photos');
$album_photos = $album_photos['photoset']['photo'];

foreach ($album_photos as $photo)
{
$all_photos[ $photo['id'] ][] = $album['title'];
}
}

// get existing photos
$query = '
SELECT id, file
FROM '.IMAGES_TABLE.'
WHERE file LIKE "'.$flickr_prefix.'%"
;';
$existing_photos = simple_hash_from_query($query, 'id', 'file');
$existing_photos = array_map(create_function('$p', 'return preg_replace("#^'.$flickr_prefix.'([0-9]+)\.([a-z]{3,4})$#i", "$1", $p);'), $existing_photos);

// remove existing photos
$duplicates = 0;
foreach ($all_photos as $id => &$photo)
{
if (in_array($id, $existing_photos))
{
unset($all_photos[$id]);
$duplicates++;
}
else
{
$photo = array(
'id' => $id,
'albums' => implode(',', $photo),
);
}
}
unset($photo);
$all_photos = array_values($all_photos);

if ($duplicates>0)
{
$page['infos'][] = '<a href="admin.php?page=batch_manager&amp;filter=prefilter-flickr">'
.l10n_dec(
'One picture is not displayed because already existing in the database.',
'%d pictures are not displayed because already existing in the database.',
$duplicates)
.'</a>';
}

$template->assign(array(
'nb_elements' => count($all_photos),
'all_elements' => json_encode($all_photos),
$template->func_combine_script([
'id' => 'plugins.flickr2piwigo.list_all',
'path' => FLICKR_PATH.'/admin/template/list_all.js',
'require' => 'jquery.ajaxmanager,jquery.jgrowl',
'load' => 'footer',
]);

$photoInfo = $flickr->people()->getPhotos($u['id']);

$template->assign([
'flickrUserId' => $u['id'],
'nb_elements' => (int)$photoInfo['total'],
'F_ACTION' => FLICKR_ADMIN . '-import&amp;action=import_set',
));
'CACHE_KEYS' => get_admin_client_cache_keys(['categories']),
]);

// get piwigo categories
$query = '
Expand Down
135 changes: 13 additions & 122 deletions admin/template/import.list_all.tpl
View file
Original file line number Diff line number Diff line change
@@ -1,125 +1,13 @@
{footer_script require='jquery.ajaxmanager,jquery.jgrowl'}
{* <!-- CATEGORIES --> *}
var categoriesCache = new CategoriesCache({
serverKey: '{$CACHE_KEYS.categories}',
serverId: '{$CACHE_KEYS._hash}',
rootUrl: '{$ROOT_URL}'
});
{footer_script}

categoriesCache.selectize(jQuery('[data-selectize=categories]'), {
filter: function(categories, options) {
if (categories.length > 0) {
jQuery("#albumSelection").show();
}

return categories;
}
});
var cacheKeysCategories = '{$CACHE_KEYS.categories}';
var cacheKeysHash = '{$CACHE_KEYS._hash}';
var rootUrl = '{$ROOT_URL}';
var photosTotal = {$nb_elements};
var flickrUserId = '{$flickrUserId}';

jQuery('[data-add-album]').pwgAddAlbum({
cache: categoriesCache,
afterSelect: function() {
jQuery("#albumSelection").show();
}
});

(function($){
/* global vars */
var all_elements = {$all_elements};
var import_done = 0;
var import_selected = {$nb_elements};
var queuedManager = $.manageAjax.create('queued', {
queue: true,
maxRequests: 1
});

/* import queue */
function performImport(photo, album, fills) {
queuedManager.add({
type: 'GET',
dataType: 'json',
url: 'ws.php',
data: {
method: 'pwg.images.addFlickr',
id: photo,
category: album,
fills: fills,
format: 'json'
},
success: function(data) {
if (data['stat'] == 'ok') {
$.jGrowl(data['result'], {
theme: 'success', life: 4000, sticky: false,
header: '{'Success'|translate}',
});
$("#photo-"+photo.id).fadeOut(function(){ $(this).remove(); });
}
else {
$.jGrowl(data['result'], {
theme: 'error', sticky: true,
header: '{'ERROR'|translate}'
});
}

import_done++;
$("#progress").html(import_done +"/"+ import_selected);

if (import_done == import_selected) {
$("#import_form").append('<input type="hidden" name="done" value="' + import_done + '">');
$("#import_form").submit();
}
},
error: function(data) {
$.jGrowl('{'an error happened'|translate|escape:'javascript'}', {
theme: 'error', sticky: true,
header: '{'ERROR'|translate}'
});
}
});
}

/* begin import */
$('#beginImport').click(function() {
$("#loader_import").fadeIn();
if ($("input[name='album_mode']:checked").val() == 'identical') {
var album = 0;
}
else {
var album = $('select[name=category] option:selected').val();
}

var fills = '';
$("input[name^='fill_']:checked").each(function() {
fills+= $(this).attr("name") +',';
});

import_selected = all_elements.length;
$("#progress").html("0/"+ import_selected);

for (var i in all_elements) {
if (album == 0) this_album = all_elements[i]['albums'];
else this_album = album;
performImport(all_elements[i]['id'], this_album, fills);
}

return false;
});

/* album mode */
$("input[name='album_mode']").change(function() {
if ($(this).val() == 'one_album') {
$("#albumSelectWrapper").slideDown();
}
else {
$("#albumSelectWrapper").slideUp();
}
});
}(jQuery));
{/footer_script}


<form action="{$F_ACTION}" method="post" id="import_form">

<fieldset>
Expand All @@ -137,7 +25,7 @@ jQuery('[data-add-album]').pwgAddAlbum({

<p>
<label><input type="radio" name="album_mode" value="identical" checked="checked"> {'Reproduce flickr albums'|translate}</label><br>
<label><input type="radio" name="album_mode" value="one_album"> {'Import all photos in this album'|translate} :</label>
<label><input type="radio" name="album_mode" value="one_album"> {'Import all photos in this album'|translate}:</label>
</p>

<p id="albumSelectWrapper" style="display:none;">
Expand All @@ -160,8 +48,11 @@ jQuery('[data-add-album]').pwgAddAlbum({
</p>

<p>
<input type="submit" name="import_set" id="beginImport" value="{'Begin transfer'|translate}" {if not $nb_elements}style="display:none;"{/if}>
<span id="loader_import" style="display:none;"><img src="admin/themes/default/images/ajax-loader.gif"> <i>{'Processing...'|translate}</i> <span id="progress"></span></span>
<button type="button" name="import_set" id="beginImport" {if not $nb_elements}style="display:none;"{/if}>{'Begin transfer'|translate}</button>
</p>
<p id="loader_import" style="display:none;">
<img src="admin/themes/default/images/ajax-loader.gif"> <i>{'Processing...'|translate}</i> <span id="progress"></span>
<button type="button" class="stop">{'Stop'|translate}</button>
</p>
</fieldset>
</form>
</form>
2 changes: 1 addition & 1 deletion admin/template/import.list_photos.tpl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -135,7 +135,7 @@ jQuery('[data-add-album]').pwgAddAlbum({
dataType: 'json',
url: 'ws.php',
data: {
method: 'pwg.images.addFlickr',
method: 'flickr2piwigo.importPhoto',
id: photo,
category: album,
fills: fills,
Expand Down
5 changes: 2 additions & 3 deletions admin/template/import.tpl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,9 +38,8 @@ jQuery('.load').click(function() {
<p>
<input type="submit" data="{$list_albums_url}" class="load" value="{'List my albums'|translate}">
<input type="submit" data="{$import_all_url}" class="load" value="{'Import all my pictures'|translate}">
<br>
<span id="loader_import" style="display:none;"><img src="admin/themes/default/images/ajax-loader.gif"> <i>{'Processing...'|translate}</i></span>
</p>
<p id="loader_import" style="display:none;"><img src="admin/themes/default/images/ajax-loader.gif"> <i>{'Processing...'|translate}</i></p>

{* <!-- ALBUMS LIST --> *}
{else if $ACTION == 'list_albums'}
Expand All @@ -60,7 +59,7 @@ jQuery('.load').click(function() {
</li>
{/foreach}
</ul>
<span id="loader_import" style="display:none;"><img src="admin/themes/default/images/ajax-loader.gif"> <i>{'Processing...'|translate}</i></span>
<p id="loader_import" style="display:none;"><img src="admin/themes/default/images/ajax-loader.gif"> <i>{'Processing...'|translate}</i></p>

{* <!-- PHOTOS LIST --> *}
{else if $ACTION == 'list_photos'}
Expand Down
Loading

0 comments on commit dd90fac

Please sign in to comment.