diff --git a/mobile/lib/entities/asset.entity.dart b/mobile/lib/entities/asset.entity.dart index 3f8c1fa74cbe7..43b9149daedd9 100644 --- a/mobile/lib/entities/asset.entity.dart +++ b/mobile/lib/entities/asset.entity.dart @@ -413,6 +413,17 @@ class Asset { static int compareById(Asset a, Asset b) => a.id.compareTo(b.id); + static int compareByLocalId(Asset a, Asset b) { + if (a.localId == null && b.localId == null) { + return 0; + } else if (a.localId == null) { + return 1; + } else if (b.localId == null) { + return -1; + } + return a.localId!.compareTo(b.localId!); + } + static int compareByChecksum(Asset a, Asset b) => a.checksum.compareTo(b.checksum); diff --git a/mobile/lib/main.dart b/mobile/lib/main.dart index 2a320fbddbe4a..3e32b8e9a61d4 100644 --- a/mobile/lib/main.dart +++ b/mobile/lib/main.dart @@ -81,6 +81,7 @@ Future initApp() async { PlatformDispatcher.instance.onError = (error, stack) { log.severe('PlatformDispatcher - Catch all', error, stack); + debugPrint("PlatformDispatcher - Catch all: $error $stack"); return true; }; diff --git a/mobile/lib/services/album.service.dart b/mobile/lib/services/album.service.dart index c2494680c7da5..b367318553d84 100644 --- a/mobile/lib/services/album.service.dart +++ b/mobile/lib/services/album.service.dart @@ -1,13 +1,8 @@ import 'dart:async'; -import 'dart:collection'; -import 'dart:io'; -import 'package:collection/collection.dart'; import 'package:flutter/foundation.dart'; import 'package:hooks_riverpod/hooks_riverpod.dart'; import 'package:immich_mobile/models/albums/album_add_asset_response.model.dart'; -import 'package:immich_mobile/entities/backup_album.entity.dart'; -import 'package:immich_mobile/services/backup.service.dart'; import 'package:immich_mobile/entities/album.entity.dart'; import 'package:immich_mobile/entities/asset.entity.dart'; import 'package:immich_mobile/entities/store.entity.dart'; @@ -28,7 +23,6 @@ final albumServiceProvider = Provider( ref.watch(userServiceProvider), ref.watch(syncServiceProvider), ref.watch(dbProvider), - ref.watch(backupServiceProvider), ), ); @@ -37,7 +31,6 @@ class AlbumService { final UserService _userService; final SyncService _syncService; final Isar _db; - final BackupService _backupService; final Logger _log = Logger('AlbumService'); Completer _localCompleter = Completer()..complete(false); Completer _remoteCompleter = Completer()..complete(false); @@ -47,7 +40,6 @@ class AlbumService { this._userService, this._syncService, this._db, - this._backupService, ); /// Checks all selected device albums for changes of albums and their assets @@ -62,60 +54,14 @@ class AlbumService { final Stopwatch sw = Stopwatch()..start(); bool changes = false; try { - final List excludedIds = - await _backupService.excludedAlbumsQuery().idProperty().findAll(); - final List selectedIds = - await _backupService.selectedAlbumsQuery().idProperty().findAll(); - if (selectedIds.isEmpty) { - final numLocal = await _db.albums.where().localIdIsNotNull().count(); - if (numLocal > 0) { - _syncService.removeAllLocalAlbumsAndAssets(); - } - return false; - } final List onDevice = await PhotoManager.getAssetPathList( hasAll: true, filterOption: FilterOptionGroup(containsPathModified: true), ); _log.info("Found ${onDevice.length} device albums"); - Set? excludedAssets; - if (excludedIds.isNotEmpty) { - if (Platform.isIOS) { - // iOS and Android device album working principle differ significantly - // on iOS, an asset can be in multiple albums - // on Android, an asset can only be in exactly one album (folder!) at the same time - // thus, on Android, excluding an album can be done by ignoring that album - // however, on iOS, it it necessary to load the assets from all excluded - // albums and check every asset from any selected album against the set - // of excluded assets - excludedAssets = await _loadExcludedAssetIds(onDevice, excludedIds); - _log.info("Found ${excludedAssets.length} assets to exclude"); - } - // remove all excluded albums - onDevice.removeWhere((e) => excludedIds.contains(e.id)); - _log.info( - "Ignoring ${excludedIds.length} excluded albums resulting in ${onDevice.length} device albums", - ); - } - final hasAll = selectedIds - .map((id) => onDevice.firstWhereOrNull((a) => a.id == id)) - .whereNotNull() - .any((a) => a.isAll); - if (hasAll) { - if (Platform.isAndroid) { - // remove the virtual "Recent" album and keep and individual albums - // on Android, the virtual "Recent" `lastModified` value is always null - onDevice.removeWhere((e) => e.isAll); - _log.info("'Recents' is selected, keeping all individual albums"); - } - } else { - // keep only the explicitly selected albums - onDevice.removeWhere((e) => !selectedIds.contains(e.id)); - _log.info("'Recents' is not selected, keeping only selected albums"); - } - changes = - await _syncService.syncLocalAlbumAssetsToDb(onDevice, excludedAssets); + + changes = await _syncService.syncLocalAlbumAssetsToDb(onDevice); _log.info("Syncing completed. Changes: $changes"); } finally { _localCompleter.complete(changes); @@ -124,21 +70,6 @@ class AlbumService { return changes; } - Future> _loadExcludedAssetIds( - List albums, - List excludedAlbumIds, - ) async { - final Set result = HashSet(); - for (AssetPathEntity a in albums) { - if (excludedAlbumIds.contains(a.id)) { - final List assets = - await a.getAssetListRange(start: 0, end: 0x7fffffffffffffff); - result.addAll(assets.map((e) => e.id)); - } - } - return result; - } - /// Checks remote albums (owned if `isShared` is false) for changes, /// updates the local database and returns `true` if there were any changes Future refreshRemoteAlbums({required bool isShared}) async { diff --git a/mobile/lib/services/hash.service.dart b/mobile/lib/services/hash.service.dart index ffc81a3445acf..a0af880983f5f 100644 --- a/mobile/lib/services/hash.service.dart +++ b/mobile/lib/services/hash.service.dart @@ -1,5 +1,6 @@ import 'dart:io'; +import 'package:collection/collection.dart'; import 'package:flutter/foundation.dart'; import 'package:hooks_riverpod/hooks_riverpod.dart'; import 'package:immich_mobile/services/background.service.dart'; @@ -20,24 +21,23 @@ class HashService { final _log = Logger('HashService'); /// Returns all assets that were successfully hashed - Future> getHashedAssets( + Stream> getHashedAssets( AssetPathEntity album, { int start = 0, int end = 0x7fffffffffffffff, - Set? excludedAssets, - }) async { + }) async* { final entities = await album.getAssetListRange(start: start, end: end); - final filtered = excludedAssets == null - ? entities - : entities.where((e) => !excludedAssets.contains(e.id)).toList(); - return _hashAssets(filtered); + entities.sortBy((e) => e.id); + await for (final assets in _hashAssets(entities)) { + yield assets; + } } /// Converts a list of [AssetEntity]s to [Asset]s including only those /// that were successfully hashed. Hashes are looked up in a DB table /// [AndroidDeviceAsset] / [IOSDeviceAsset] by local id. Only missing /// entries are newly hashed and added to the DB table. - Future> _hashAssets(List assetEntities) async { + Stream> _hashAssets(List assetEntities) async* { const int batchFileCount = 128; const int batchDataSize = 1024 * 1024 * 1024; // 1GB @@ -46,7 +46,7 @@ class HashService { .toList(); final List hashes = await _lookupHashes(ids); final List toAdd = []; - final List toHash = []; + final List> toHash = []; int bytes = 0; @@ -70,23 +70,26 @@ class HashService { continue; } bytes += await file.length(); - toHash.add(file.path); + toHash.add({file.path: assetEntities[i]}); final deviceAsset = Platform.isAndroid ? AndroidDeviceAsset(id: ids[i] as int, hash: const []) : IOSDeviceAsset(id: ids[i] as String, hash: const []); toAdd.add(deviceAsset); hashes[i] = deviceAsset; if (toHash.length == batchFileCount || bytes >= batchDataSize) { - await _processBatch(toHash, toAdd); + await for (final batch in _processBatch(toHash, toAdd)) { + yield batch; + } toAdd.clear(); toHash.clear(); bytes = 0; } } if (toHash.isNotEmpty) { - await _processBatch(toHash, toAdd); + await for (final batch in _processBatch(toHash, toAdd)) { + yield batch; + } } - return _mapAllHashedAssets(assetEntities, hashes); } /// Lookup hashes of assets by their local ID @@ -97,29 +100,34 @@ class HashService { /// Processes a batch of files and saves any successfully hashed /// values to the DB table. - Future _processBatch( - final List toHash, + Stream> _processBatch( + final List> toHash, final List toAdd, - ) async { - final hashes = await _hashFiles(toHash); + ) async* { + final List validLocalAssets = []; + final hashes = await _hashFiles(toHash.map((e) => e.keys.first).toList()); bool anyNull = false; for (int j = 0; j < hashes.length; j++) { if (hashes[j]?.length == 20) { toAdd[j].hash = hashes[j]!; + validLocalAssets.add(Asset.local(toHash[j].values.first, hashes[j]!)); } else { _log.warning("Failed to hash file ${toHash[j]}, skipping"); anyNull = true; } } + final validHashes = anyNull ? toAdd.where((e) => e.hash.length == 20).toList(growable: false) : toAdd; + await _db.writeTxn( () => Platform.isAndroid ? _db.androidDeviceAssets.putAll(validHashes.cast()) : _db.iOSDeviceAssets.putAll(validHashes.cast()), ); _log.fine("Hashed ${validHashes.length}/${toHash.length} assets"); + yield validLocalAssets; } /// Hashes the given files and returns a list of the same length @@ -132,20 +140,6 @@ class HashService { } return hashes; } - - /// Converts [AssetEntity]s that were successfully hashed to [Asset]s - List _mapAllHashedAssets( - List assets, - List hashes, - ) { - final List result = []; - for (int i = 0; i < assets.length; i++) { - if (hashes[i] != null && hashes[i]!.hash.isNotEmpty) { - result.add(Asset.local(assets[i], hashes[i]!.hash)); - } - } - return result; - } } final hashServiceProvider = Provider( diff --git a/mobile/lib/services/sync.service.dart b/mobile/lib/services/sync.service.dart index 8ec56e925f7f8..8b19426c48fd6 100644 --- a/mobile/lib/services/sync.service.dart +++ b/mobile/lib/services/sync.service.dart @@ -68,10 +68,9 @@ class SyncService { /// Syncs all device albums and their assets to the database /// Returns `true` if there were any changes Future syncLocalAlbumAssetsToDb( - List onDevice, [ - Set? excludedAssets, - ]) => - _lock.run(() => _syncLocalAlbumAssetsToDb(onDevice, excludedAssets)); + List onDevice, + ) => + _lock.run(() => _syncLocalAlbumAssetsToDb(onDevice)); /// returns all Asset IDs that are not contained in the existing list List sharedAssetsToRemove( @@ -492,9 +491,8 @@ class SyncService { /// Syncs all device albums and their assets to the database /// Returns `true` if there were any changes Future _syncLocalAlbumAssetsToDb( - List onDevice, [ - Set? excludedAssets, - ]) async { + List onDevice, + ) async { onDevice.sort((a, b) => a.id.compareTo(b.id)); final inDb = await _db.albums.where().localIdIsNotNull().sortByLocalId().findAll(); @@ -510,10 +508,8 @@ class SyncService { album, deleteCandidates, existing, - excludedAssets, ), - onlyFirst: (AssetPathEntity ape) => - _addAlbumFromDevice(ape, existing, excludedAssets), + onlyFirst: (AssetPathEntity ape) => _addAlbumFromDevice(ape, existing), onlySecond: (Album a) => _removeAlbumFromDb(a, deleteCandidates), ); _log.fine( @@ -545,16 +541,13 @@ class SyncService { Album album, List deleteCandidates, List existing, [ - Set? excludedAssets, bool forceRefresh = false, ]) async { if (!forceRefresh && !await _hasAssetPathEntityChanged(ape, album)) { _log.fine("Local album ${ape.name} has not changed. Skipping sync."); return false; } - if (!forceRefresh && - excludedAssets == null && - await _syncDeviceAlbumFast(ape, album)) { + if (!forceRefresh && await _syncDeviceAlbumFast(ape, album)) { return true; } @@ -562,66 +555,72 @@ class SyncService { final inDb = await album.assets .filter() .ownerIdEqualTo(Store.get(StoreKey.currentUser).isarId) - .sortByChecksum() + .sortByLocalId() .findAll(); - assert(inDb.isSorted(Asset.compareByChecksum), "inDb not sorted!"); + assert(inDb.isSorted(Asset.compareByLocalId), "inDb not sorted!"); final int assetCountOnDevice = await ape.assetCountAsync; - final List onDevice = - await _hashService.getHashedAssets(ape, excludedAssets: excludedAssets); - _removeDuplicates(onDevice); - // _removeDuplicates sorts `onDevice` by checksum - final (toAdd, toUpdate, toDelete) = _diffAssets(onDevice, inDb); - if (toAdd.isEmpty && - toUpdate.isEmpty && - toDelete.isEmpty && - album.name == ape.name && - ape.lastModified != null && - album.modifiedAt.isAtSameMomentAs(ape.lastModified!)) { - // changes only affeted excluded albums + + await for (final onDevice in _hashService.getHashedAssets(ape)) { + _removeDuplicates(onDevice); + // _removeDuplicates sorts `onDevice` by checksum + final (toAdd, toUpdate, toDelete) = _diffAssets( + onDevice, + inDb, + compare: Asset.compareByLocalId, + ); + + if (toAdd.isEmpty && + toUpdate.isEmpty && + toDelete.isEmpty && + album.name == ape.name && + ape.lastModified != null && + album.modifiedAt.isAtSameMomentAs(ape.lastModified!)) { + // changes only affeted excluded albums + _log.fine( + "Only excluded assets in local album ${ape.name} changed. Stopping sync.", + ); + if (assetCountOnDevice != + _db.eTags.getByIdSync(ape.eTagKeyAssetCount)?.assetCount) { + await _db.writeTxn( + () => _db.eTags.put( + ETag(id: ape.eTagKeyAssetCount, assetCount: assetCountOnDevice), + ), + ); + } + return false; + } _log.fine( - "Only excluded assets in local album ${ape.name} changed. Stopping sync.", + "Syncing local album ${ape.name}. ${toAdd.length} assets to add, ${toUpdate.length} to update, ${toDelete.length} to delete", ); - if (assetCountOnDevice != - _db.eTags.getByIdSync(ape.eTagKeyAssetCount)?.assetCount) { - await _db.writeTxn( - () => _db.eTags.put( + final (existingInDb, updated) = await _linkWithExistingFromDb(toAdd); + _log.fine( + "Linking assets to add with existing from db. ${existingInDb.length} existing, ${updated.length} to update", + ); + deleteCandidates.addAll(toDelete); + existing.addAll(existingInDb); + album.name = ape.name; + album.modifiedAt = ape.lastModified ?? DateTime.now(); + if (album.thumbnail.value != null && + toDelete.contains(album.thumbnail.value)) { + album.thumbnail.value = null; + } + try { + await _db.writeTxn(() async { + await _db.assets.putAll(updated); + await _db.assets.putAll(toUpdate); + await album.assets + .update(link: existingInDb + updated, unlink: toDelete); + await _db.albums.put(album); + album.thumbnail.value ??= await album.assets.filter().findFirst(); + await album.thumbnail.save(); + await _db.eTags.put( ETag(id: ape.eTagKeyAssetCount, assetCount: assetCountOnDevice), - ), - ); + ); + }); + _log.info("Synced changes of local album ${ape.name} to DB"); + } on IsarError catch (e) { + _log.severe("Failed to update synced album ${ape.name} in DB", e); } - return false; - } - _log.fine( - "Syncing local album ${ape.name}. ${toAdd.length} assets to add, ${toUpdate.length} to update, ${toDelete.length} to delete", - ); - final (existingInDb, updated) = await _linkWithExistingFromDb(toAdd); - _log.fine( - "Linking assets to add with existing from db. ${existingInDb.length} existing, ${updated.length} to update", - ); - deleteCandidates.addAll(toDelete); - existing.addAll(existingInDb); - album.name = ape.name; - album.modifiedAt = ape.lastModified ?? DateTime.now(); - if (album.thumbnail.value != null && - toDelete.contains(album.thumbnail.value)) { - album.thumbnail.value = null; - } - try { - await _db.writeTxn(() async { - await _db.assets.putAll(updated); - await _db.assets.putAll(toUpdate); - await album.assets - .update(link: existingInDb + updated, unlink: toDelete); - await _db.albums.put(album); - album.thumbnail.value ??= await album.assets.filter().findFirst(); - await album.thumbnail.save(); - await _db.eTags.put( - ETag(id: ape.eTagKeyAssetCount, assetCount: assetCountOnDevice), - ); - }); - _log.info("Synced changes of local album ${ape.name} to DB"); - } on IsarError catch (e) { - _log.severe("Failed to update synced album ${ape.name} in DB", e); } return true; @@ -649,26 +648,27 @@ class SyncService { if (modified == null) { return false; } - final List newAssets = await _hashService.getHashedAssets(modified); - if (totalOnDevice != lastKnownTotal + newAssets.length) { - return false; - } - album.modifiedAt = ape.lastModified ?? DateTime.now(); - _removeDuplicates(newAssets); - final (existingInDb, updated) = await _linkWithExistingFromDb(newAssets); - try { - await _db.writeTxn(() async { - await _db.assets.putAll(updated); - await album.assets.update(link: existingInDb + updated); - await _db.albums.put(album); - await _db.eTags - .put(ETag(id: ape.eTagKeyAssetCount, assetCount: totalOnDevice)); - }); - _log.info("Fast synced local album ${ape.name} to DB"); - } on IsarError catch (e) { - _log.severe("Failed to fast sync local album ${ape.name} to DB", e); - return false; + await for (final newAssets in _hashService.getHashedAssets(modified)) { + if (totalOnDevice != lastKnownTotal + newAssets.length) { + return false; + } + album.modifiedAt = ape.lastModified ?? DateTime.now(); + _removeDuplicates(newAssets); + final (existingInDb, updated) = await _linkWithExistingFromDb(newAssets); + try { + await _db.writeTxn(() async { + await _db.assets.putAll(updated); + await album.assets.update(link: existingInDb + updated); + await _db.albums.put(album); + await _db.eTags + .put(ETag(id: ape.eTagKeyAssetCount, assetCount: totalOnDevice)); + }); + _log.info("Fast synced local album ${ape.name} to DB"); + } on IsarError catch (e) { + _log.severe("Failed to fast sync local album ${ape.name} to DB", e); + return false; + } } return true; @@ -678,29 +678,28 @@ class SyncService { /// assets already existing in the database to the list of `existing` assets Future _addAlbumFromDevice( AssetPathEntity ape, - List existing, [ - Set? excludedAssets, - ]) async { + List existing, + ) async { _log.info("Syncing a new local album to DB: ${ape.name}"); final Album a = Album.local(ape); - final assets = - await _hashService.getHashedAssets(ape, excludedAssets: excludedAssets); - _removeDuplicates(assets); - final (existingInDb, updated) = await _linkWithExistingFromDb(assets); - _log.info( - "${existingInDb.length} assets already existed in DB, to upsert ${updated.length}", - ); - await upsertAssetsWithExif(updated); - existing.addAll(existingInDb); - a.assets.addAll(existingInDb); - a.assets.addAll(updated); - final thumb = existingInDb.firstOrNull ?? updated.firstOrNull; - a.thumbnail.value = thumb; - try { - await _db.writeTxn(() => _db.albums.store(a)); - _log.info("Added a new local album to DB: ${ape.name}"); - } on IsarError catch (e) { - _log.severe("Failed to add new local album ${ape.name} to DB", e); + await for (final assets in _hashService.getHashedAssets(ape)) { + _removeDuplicates(assets); + final (existingInDb, updated) = await _linkWithExistingFromDb(assets); + _log.info( + "${existingInDb.length} assets already existed in DB, to upsert ${updated.length}", + ); + await upsertAssetsWithExif(updated); + existing.addAll(existingInDb); + a.assets.addAll(existingInDb); + a.assets.addAll(updated); + final thumb = existingInDb.firstOrNull ?? updated.firstOrNull; + a.thumbnail.value = thumb; + try { + await _db.writeTxn(() => _db.albums.store(a)); + _log.info("Added a new local album to DB: ${ape.name}"); + } on IsarError catch (e) { + _log.severe("Failed to add new local album ${ape.name} to DB", e); + } } }